Skip to content
This repository was archived by the owner on Jun 12, 2019. It is now read-only.

Commit 912f1bb

Browse files
Caroline Twiggkmussel
Caroline Twigg
authored and
kmussel
committed
download models (#50)
* request for list models * adding model command to usage and readme * dispatch model function * list models * add download models function to header * download model * convert json array of tags to comma separated string * dispatch model download * add download url * checking if download was successful * add download to file and resolve path to Models class * model tags as json array to comma delimited string * download the model to a file * resolve output path string whether it is a dir or a file * missing side carrot * breakout nested ifs * update display name to name * Fix model download directory check
1 parent 42676ae commit 912f1bb

9 files changed

Lines changed: 188 additions & 38 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ You can install [https://brew.sh/](Homebrew), or use the same process as Linux.
4848
skafos remote info [<project_token>]
4949
skafos orgs [<name>] [--set-default]
5050
skafos whoami
51-
skafos models (list [<name>] [--deployment <deployment_id] [--job <job_id>]|download <name> [-o <output_path>]) [--project <token>] [--tag <tag>] [-v version]
51+
skafos models (list [<name>] [--deployment <deployment_id>] [--job <job_id>]|download <name> [-o <output_path>]) [--project <token>] [--tag <tag>] [-v version]
5252
skafos -h | --help
5353
skafos --version
5454
## Commands:

‎build_envs/production_env.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44
#include <stdlib.h>
55
#include <string>
66

7-
const std::string ENVIRONMENT = "production";
8-
const std::string CLIENT_ID = "0cbe0e8754f1492cac415f73cea669bb";
9-
const std::string API_URL = "https://api.metismachine.io/v1";
10-
const std::string LOGGING_URL = "https://api.metismachine.io/logs/";
11-
const std::string VASI_URL = "https://vasi.metismachine.io/";
7+
const std::string ENVIRONMENT = "production";
8+
const std::string CLIENT_ID = "0cbe0e8754f1492cac415f73cea669bb";
9+
const std::string API_URL = "https://api.metismachine.io/v1";
10+
const std::string LOGGING_URL = "https://api.metismachine.io/logs/";
11+
const std::string VASI_URL = "https://vasi.metismachine.io/";
12+
const std::string DOWNLOAD_URL = "http://download.metismachine.io/v1";
1213

1314
#endif

‎build_envs/staging_env.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44
#include <stdlib.h>
55
#include <string>
66

7-
const std::string ENVIRONMENT = "staging";
8-
const std::string CLIENT_ID = "f1f6e59f3f6f8ffecde29d34ad18f673";
9-
const std::string API_URL = "https://api.metis.wtf/v1";
10-
const std::string LOGGING_URL = "https://api.metis.wtf/logs/";
11-
const std::string VASI_URL = "https://vasi.metis.wtf/";
7+
const std::string ENVIRONMENT = "staging";
8+
const std::string CLIENT_ID = "f1f6e59f3f6f8ffecde29d34ad18f673";
9+
const std::string API_URL = "https://api.metis.wtf/v1";
10+
const std::string LOGGING_URL = "https://api.metis.wtf/logs/";
11+
const std::string VASI_URL = "https://vasi.metis.wtf/";
12+
const std::string DOWNLOAD_URL = "http://download.metis.wtf/v1";
1213

1314
#endif

‎src/dispatch/dispatch.cpp‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,30 @@ void whoami() {
389389
Whoami::information();
390390
}
391391

392+
void download_models(int argc, char **argv, int cmd_index, map<string, int> flags, map<string, string> params){
393+
string output_path = ".";
394+
string project_token = ".";
395+
396+
std::string model_name = string(argv[3]);
397+
params.insert(std::pair<string,string> ("name", model_name));
398+
399+
int project_index = flags.find("--project")->second;
400+
if (project_index != -1){
401+
if (project_index + 1 < argc){
402+
project_token = argv[project_index + 1];
403+
}
404+
}
405+
406+
int output_index = flags.find("-o")->second;
407+
if (output_index != -1){
408+
if (output_index + 1 < argc){
409+
output_path = argv[output_index + 1];
410+
}
411+
}
412+
413+
Models::download(project_token, params, output_path);
414+
}
415+
392416
void list_models (int argc, char **argv, int cmd_index, map<string, int> flags, map<string, string> params){
393417
string project_token = ".";
394418
string model_name = "";
@@ -397,7 +421,7 @@ void list_models (int argc, char **argv, int cmd_index, map<string, int> flags,
397421

398422
if (flags.find(string(argv[3])) == flags.end()){
399423
std::string model_name = string(argv[3]);
400-
params.insert(std::pair<string,string> ("display_name", model_name));
424+
params.insert(std::pair<string,string> ("name", model_name));
401425
}
402426

403427
int deployment_index = flags.find("--deployment")->second;
@@ -424,7 +448,7 @@ void list_models (int argc, char **argv, int cmd_index, map<string, int> flags,
424448
}
425449

426450
if (params.find("tag") != params.end() | params.find("version") != params.end()){
427-
if (params.find("display_name") != params.end()){
451+
if (params.find("name") != params.end()){
428452
Models::list(project_token, params);
429453
} else {
430454
console::error("A model name is required in order to list models by tag or version.");
@@ -459,10 +483,9 @@ void models(int argc, char **argv, int cmd_index) {
459483

460484
action = string(argv[2]);
461485
if (action.compare("list") == 0){
462-
console::debug("list models");
463486
list_models(argc, argv, cmd_index, flags, params);
464487
} else if (action.compare("download") == 0){
465-
console::info("download models coming soon.");
488+
download_models(argc, argv, cmd_index, flags, params);
466489
}
467490
}
468491

‎src/file/file.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,11 @@ bool FileManager::dir_exists(string path) {
126126

127127
bool FileManager::is_dir(string path) {
128128
struct stat buffer;
129-
if(stat(path.c_str(), &buffer) != 0) {
130-
return true;
131-
}
132-
133-
return S_ISDIR(buffer.st_mode);
129+
if(stat(path.c_str(), &buffer) == 0) {
130+
return S_ISDIR(buffer.st_mode);
131+
}
132+
133+
return false;
134134
}
135135

136136
bool FileManager::is_dot_dir(string path) {

‎src/models/models.cpp‎

Lines changed: 132 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,8 @@ void Models::list(std::string project_token, std::map<std::string, std::string>
1818
console::info("Your models: ");
1919
auto models = json.array_items();
2020
for (int i = 0; i < models.size(); i++) {
21-
auto tags = models[i]["tags"].array_items();
22-
std::string model_tags;
23-
if (tags.size() == 0){
24-
model_tags = "nil";
25-
} else {
26-
for (int j = 0; j < tags.size(); j++) {
27-
if(j == tags.size() - 1){
28-
model_tags = model_tags + " " + tags[j].string_value();
29-
} else {
30-
model_tags = model_tags + " " + tags[j].string_value() + ",";
31-
}
32-
}
33-
34-
}
21+
Json tags = models[i]["tags"];
22+
std::string model_tags = tags_to_string(tags);
3523

3624
console::info(" Name: " + models[i]["display_name"].string_value() +
3725
"\n Version: " + models[i]["version"].string_value() +
@@ -40,4 +28,133 @@ void Models::list(std::string project_token, std::map<std::string, std::string>
4028
} else {
4129
console::error("There was an error listing your models \n");
4230
}
43-
}
31+
}
32+
33+
void Models::download(std::string project_token, std::map<std::string, std::string> params, std::string output_path){
34+
if(project_token.compare(".") == 0){
35+
project_token = PROJECT_TOKEN;
36+
}
37+
38+
RestClient::Response resp = Request::list_models(project_token, params);
39+
std::string err;
40+
Json json = Json::parse(resp.body, err);
41+
42+
int status_code = resp.code;
43+
std::string error_message = json["message"].string_value();
44+
45+
Json selected_model;
46+
47+
if (not(Helpers::success(status_code))) {
48+
console::error("There was an error retreiving your model for download: " + std::to_string(status_code) + " " + error_message + "\n");
49+
} else if (json.is_array()) {
50+
auto models = json.array_items();
51+
string list_size = std::to_string(models.size());
52+
53+
if(stoi(list_size) == 1){
54+
selected_model = models[0];
55+
} else{
56+
cout << "Please select the model you would like to download." << endl;
57+
int opt_iter = 1;
58+
for(int i = 0; i < models.size(); i++){
59+
Json tags = models[i]["tags"];
60+
std::string model_tags = tags_to_string(tags);
61+
62+
cout << std::to_string(opt_iter) << ". " << models[i]["display_name"].string_value() << endl;
63+
cout << "Version: " + models[i]["version"].string_value() << endl;
64+
cout << "Tags: " + model_tags << endl;
65+
cout << endl;
66+
opt_iter++;
67+
}
68+
cout << "Enter your choice and press return: ";
69+
string opt_select;
70+
cin >> opt_select;
71+
auto idx = stoi(opt_select) - 1;
72+
if(idx >= models.size()) {
73+
cin.clear();
74+
cin.ignore(1, '\n');
75+
cout << "Invalid choice. Valid numbers are [1-" << list_size << "].";
76+
return;
77+
} else {
78+
selected_model = models[idx];
79+
}
80+
}
81+
std::string display_name = selected_model["display_name"].string_value();
82+
std::string version = selected_model["version"].string_value();
83+
84+
Models::download_to_file(project_token, display_name, version, output_path);
85+
86+
} else {
87+
console::error("There was an error downloading your model \n");
88+
}
89+
}
90+
91+
std::string Models::tags_to_string(Json tags){
92+
auto tags_list = tags.array_items();
93+
std::string model_tags;
94+
if (tags_list.size() == 0){
95+
model_tags = "nil";
96+
} else {
97+
for (int j = 0; j < tags_list.size(); j++) {
98+
if(j == tags_list.size() - 1){
99+
model_tags = model_tags + " " + tags_list[j].string_value();
100+
} else {
101+
model_tags = model_tags + " " + tags_list[j].string_value() + ",";
102+
}
103+
}
104+
}
105+
return model_tags;
106+
}
107+
108+
void Models::download_to_file(std::string project_token, std::string display_name, std::string version, std::string output_path){
109+
110+
std::string model_url = DOWNLOAD_URL + "/projects/" + project_token + "/models?name=" + display_name + "&version=" + version;
111+
112+
std::string output_file = display_name;
113+
std::string download_path;
114+
115+
try {
116+
download_path = resolve_download_path(output_path, output_file);
117+
console::debug("Download path: " + download_path);
118+
Request::download(model_url, download_path);
119+
120+
YAML::Node model_error = YAML::LoadFile(download_path);
121+
if (model_error["Error"]){
122+
console::error("There was an error downloading your model to " + download_path + ": " + model_error["Error"].as<std::string>());
123+
} else{
124+
console::error("There was an error downloading your model to " + download_path);
125+
}
126+
FileManager::delete_file(download_path);
127+
} catch (YAML::ParserException& e){
128+
console::info("Model successfully downloaded to " + download_path);
129+
} catch(std::string &error) {
130+
console::error("Error: " + error);
131+
} catch (...) {
132+
console::error("There was an error downloading your model to " + download_path);
133+
}
134+
}
135+
136+
std::string Models::resolve_download_path(std::string output_path, std::string output_file) {
137+
std::string download_path;
138+
139+
if (FileManager::is_dir(output_path)){
140+
char ch = output_path.back();
141+
if (ch == '/') {
142+
download_path = output_path + output_file;
143+
} else {
144+
download_path = output_path + "/" + output_file;
145+
}
146+
} else {
147+
std::size_t found = output_path.rfind("/");
148+
if (found == std::string::npos) {
149+
download_path = FileManager::resolve_path(output_path);
150+
} else {
151+
string dir = output_path.substr(0, found);
152+
if (FileManager::is_dir(dir)){
153+
download_path = output_path;
154+
} else {
155+
throw "Directory does not exist: " + dir;
156+
}
157+
}
158+
}
159+
return download_path;
160+
}

‎src/models/models.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,23 @@
44
#include "common.h"
55
#include "project_env/project_env.h"
66
#include "request/request.h"
7+
#include "env/env.h"
8+
#include "file/file.h"
9+
#include <curl/curl.h>
10+
#include "yaml-cpp/yaml.h"
711

812
using namespace std;
913
using namespace json11;
1014

1115
class Models {
1216
public:
1317
static void list(std::string project_token, std::map<string, string> params);
18+
static void download(std::string project_token, std::map<string,string> params, std::string output_path);
19+
20+
private:
21+
static void download_to_file(std::string project_token, std::string display_name, std::string version, std::string output_path);
22+
static string resolve_download_path(std::string output_path, std::string output_file);
23+
static string tags_to_string(Json model_tags);
1424
};
1525

1626
#endif

‎src/request/request.cpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -381,8 +381,6 @@ RestClient::Response Request::_list_models(std::string project_token, std::map<s
381381
}
382382
}
383383

384-
cout << request_params << endl;
385-
386384
std::string uri = PROJECT_URL + "/" + project_token + MODELS_URL + "?" + request_params;
387385

388386
return this->connection->get(uri);

‎src/usage.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ R"(
1515
skafos remote info [<project_token>]
1616
skafos orgs [<name>] [--set-default]
1717
skafos whoami
18-
skafos models (list [<name>] [--deployment <deployment_id] [--job <job_id>]|download <name> [-o <output_path>]) [--project <token>] [--tag <tag>] [-v version]
18+
skafos models (list [<name>] [--deployment <deployment_id>] [--job <job_id>]|download <name> [-o <output_path>]) [--project <token>] [--tag <tag>] [-v version]
1919
skafos -h | --help
2020
skafos --version
2121
Commands:

0 commit comments

Comments
 (0)