From 745caa185f2afe26e613131efa8f354455632fd4 Mon Sep 17 00:00:00 2001
From: wangyuansheng <wangyuansheng>
Date: Fri, 14 Oct 2016 21:49:34 +0800
Subject: [PATCH 1/8] fetch repository information

and do some other thins:
1. remove unused variable
2. fix the final break
---
 web/lua/opmserver.lua | 59 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 55 insertions(+), 4 deletions(-)

diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 0a5a6f4..72434e9 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -364,6 +364,12 @@ function _M.do_upload()
         end
     end
 
+    -- get the repository's information
+    do
+      local repo_info = query_github_repository(ctx, pkg_name)
+      ctx.starred = repo_info.stargazers_count
+    end
+
     do
         req_read_body()
 
@@ -486,7 +492,7 @@ function db_update_user_info(ctx, user_info, user_id)
 end
 
 
-function query_github_user(ctx, token)
+function query_github_user(ctx)
     local path = "/user"
     local res = query_github(ctx, path)
 
@@ -528,6 +534,50 @@ function query_github_user(ctx, token)
 end
 
 
+function query_github_repository(ctx, repository)
+    local account = ctx.account
+    local path = "/repos/" .. "account .. "/" .. repository"
+
+    local res = query_github(ctx, path)
+
+    local scopes = res.headers["X-OAuth-Scopes"]
+
+    if not scopes or not str_find(scopes, "user:email", nil, true) then
+        return log_and_out_err(ctx, 403,
+                               "personal access token lacking ",
+                               "the user:email scope: ", scopes)
+    end
+
+    if #scopes > #"read:org, user:email" then
+        return log_and_out_err(ctx, 403,
+                               "personal access token is too permissive; ",
+                               "only the scopes user:email and read:org ",
+                               "should be allowed.")
+    end
+
+    -- say(cjson.encode(res.headers))
+
+    local json = res.body
+
+    -- say("user json: ", json)
+
+    local data, err = decode_json(json)
+    if not data then
+        return log_and_out_err(ctx, 502, "failed to parse repos json: ",
+                               err, " (", json, ")")
+    end
+
+    local login = data.owner.login
+    if not login then
+        return log_and_out_err(ctx, 502,
+                               "login name cannot found in the ",
+                               "github /repos API call: ", json)
+    end
+
+    return data, scopes
+end
+
+
 function db_insert_org_info(ctx, org_info)
     local o = org_info
     local q = quote_sql_str
@@ -675,7 +725,7 @@ do
     function query_github(ctx, path)
         local httpc = ctx.httpc
         local auth = ctx.auth
-        
+
         if not httpc then
             httpc = http.new()
             ctx.httpc = httpc
@@ -728,8 +778,9 @@ do
                 goto continue
             end
 
-            break
-
+            do
+              break
+            end
             ::continue::
         end
 

From 62d9bcac4d1d2613cd15bfb7235f562eadbd9948 Mon Sep 17 00:00:00 2001
From: wangyuansheng <wangyuansheng>
Date: Fri, 14 Oct 2016 22:46:15 +0800
Subject: [PATCH 2/8] record the stargazers_count of repository

---
 init.sql              | 1 +
 web/lua/opmserver.lua | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/init.sql b/init.sql
index 12566e1..9443f8a 100644
--- a/init.sql
+++ b/init.sql
@@ -97,6 +97,7 @@ create table uploads (
     size integer not null,
     package integer references packages(id),
     abstract text,
+    stargazers_count integer not null,
 
     version_v integer[] not null,
     version_s varchar(128) not null,
diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 72434e9..9fdc1af 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -365,9 +365,10 @@ function _M.do_upload()
     end
 
     -- get the repository's information
+    local stargazers_count
     do
       local repo_info = query_github_repository(ctx, pkg_name)
-      ctx.starred = repo_info.stargazers_count
+      stargazers_count = repo_info.stargazers_count
     end
 
     do
@@ -395,7 +396,7 @@ function _M.do_upload()
     -- insert the new uploaded task to the uplaods database.
 
     local sql1 = "insert into uploads (uploader, size, package, orig_checksum, "
-                  .. "version_v, version_s, client_addr"
+                  .. "version_v, version_s, client_addr, stargazers_count"
 
     local sql2 = ""
     if login ~= account then
@@ -408,6 +409,7 @@ function _M.do_upload()
                  .. ", " .. ver_v
                  .. ", " .. quote_sql_str(pkg_version)
                  .. ", " .. quote_sql_str(ngx_var.remote_addr)
+                 .. ", " .. quote_sql_str(stargazers_count)
 
     local sql4
     if login ~= account then
@@ -536,7 +538,7 @@ end
 
 function query_github_repository(ctx, repository)
     local account = ctx.account
-    local path = "/repos/" .. "account .. "/" .. repository"
+    local path = "/repos/" .. account .. "/" .. repository
 
     local res = query_github(ctx, path)
 

From 390e213e16bddd90de87a53f31c670bd4f37678f Mon Sep 17 00:00:00 2001
From: wangyuansheng <wangyuansheng>
Date: Sat, 15 Oct 2016 00:55:06 +0800
Subject: [PATCH 3/8] show the stars

---
 web/lua/opmserver.lua | 37 ++++++++++++-------------------------
 1 file changed, 12 insertions(+), 25 deletions(-)

diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 58365e1..76538e2 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -1517,9 +1517,11 @@ do
 
         local sql = "select abstract, package_name, orgs.login as org_name"
                     .. ", users.login as uploader_name"
+                    .. ", tmp.stargazers_count as stargazers_count"
                     .. " from (select last(abstract) as abstract"
                     .. ", package_name, org_account, uploader"
                     .. ", ts_rank_cd(last(ts_idx), last(q), 1) as rank"
+                    .. ", max(stargazers_count) as stargazers_count"
                     .. " from uploads, plainto_tsquery("
                     .. quote_sql_str(query) .. ") q"
                     .. " where indexed = true and ts_idx @@ q"
@@ -1539,11 +1541,17 @@ do
 
         tab_clear(results)
 
-        local i = 0
+        local i = 1
+        local show_pattern = "%-40s  %6s  %s\n"
+        results[i] = string.format(show_pattern,
+                    "NAME",
+                    "STARS", "DESCRIPTION")
+
         for _, row in ipairs(rows) do
             local uploader = row.uploader_name
             local org = row.org_name
             local pkg = row.package_name
+            local star = tostring(row.stargazers_count)
 
             local account
             if org and org ~= ngx_null then
@@ -1554,30 +1562,9 @@ do
             end
 
             i = i + 1
-            results[i] = account
-
-            i = i + 1
-            results[i] = "/"
-
-            i = i + 1
-            results[i] = pkg
-
-            local len = #account + #pkg + 1
-
-            if len < 50 then
-                len = 50 - len
-            else
-                len = 4
-            end
-
-            i = i + 1
-            results[i] = str_rep(" ", len)
-
-            i = i + 1
-            results[i] = row.abstract
-
-            i = i + 1
-            results[i] = "\n"
+            results[i] = string.format(show_pattern,
+                    account .. "/" .. pkg,
+                    star, row.abstract)
         end
 
         ngx_print(results)

From 19f0e8ca487962c9ba5d9d6807734f3d88c4a519 Mon Sep 17 00:00:00 2001
From: wangyuansheng <wangyuansheng>
Date: Sat, 15 Oct 2016 01:00:43 +0800
Subject: [PATCH 4/8] local variable

---
 web/lua/opmserver.lua | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 76538e2..647ad79 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -19,6 +19,7 @@ local send_email = email.send_mail
 local re_find = ngx.re.find
 local re_match = ngx.re.match
 local str_find = string.find
+local str_format = string.format
 local ngx_var = ngx.var
 local say = ngx.say
 local req_read_body = ngx.req.read_body
@@ -1543,9 +1544,8 @@ do
 
         local i = 1
         local show_pattern = "%-40s  %6s  %s\n"
-        results[i] = string.format(show_pattern,
-                    "NAME",
-                    "STARS", "DESCRIPTION")
+        results[i] = str_format(show_pattern,
+                    "NAME", "STARS", "DESCRIPTION")
 
         for _, row in ipairs(rows) do
             local uploader = row.uploader_name
@@ -1562,9 +1562,8 @@ do
             end
 
             i = i + 1
-            results[i] = string.format(show_pattern,
-                    account .. "/" .. pkg,
-                    star, row.abstract)
+            results[i] = str_format(show_pattern,
+                    account .. "/" .. pkg, star, row.abstract)
         end
 
         ngx_print(results)

From 5a77f89b4ca78527d10b951e114a1f2cfaa91f63 Mon Sep 17 00:00:00 2001
From: wangyuansheng <wangyuansheng>
Date: Sat, 15 Oct 2016 01:04:40 +0800
Subject: [PATCH 5/8] syn limit GitHub

---
 web/lua/opmserver.lua | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 647ad79..5578168 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -568,6 +568,29 @@ end
 
 
 function query_github_repository(ctx, repository)
+    -- limit github accesses globally
+    local zone = "global_github_limit"
+    local lim, err = limit_req.new(zone, 1, 5)
+    if not lim then
+        return log_and_out_err(ctx, 500, "failed to new resty.limit.req: ", err)
+    end
+
+    local delay, err = lim:incoming("github", true)
+    if not delay then
+        if err == "rejected" then
+            return log_and_out_err(ctx, 503, "server too busy");
+        end
+        return log_and_out_err(ctx, 500, "failed to limit req: ", err)
+    end
+
+    if delay >= 0.001 then
+        local excess = err
+        ngx.log(ngx.WARN, "delaying github request, excess: ", excess,
+                " by zone ", zone)
+        ngx.sleep(delay)
+    end
+
+    -- "/repos/{owner}/{repo}"
     local account = ctx.account
     local path = "/repos/" .. account .. "/" .. repository
 

From 7d0df951e0bd069a5fa1208810d309004a25e075 Mon Sep 17 00:00:00 2001
From: yuansheng <membphis@gmail.com>
Date: Thu, 27 Oct 2016 23:15:19 +0800
Subject: [PATCH 6/8] remove unused variable

---
 web/lua/opmserver.lua | 2 --
 1 file changed, 2 deletions(-)

diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 5578168..72d483e 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -28,8 +28,6 @@ local encode_json = cjson.encode
 local req_method = ngx.req.get_method
 local req_body_file = ngx.req.get_body_file
 local os_exec = os.execute
-local io_open = io.open
-local io_close = io.close
 local set_quote_sql_str = ndk.set_var.set_quote_pgsql_str
 local assert = assert
 local sub = string.sub

From e1ca1d5cd45dde8c0ab10af2e303119f8b3afb62 Mon Sep 17 00:00:00 2001
From: yuansheng <membphis@gmail.com>
Date: Thu, 27 Oct 2016 23:48:15 +0800
Subject: [PATCH 7/8] move it to update.sql for updating

---
 init.sql | 1 -
 1 file changed, 1 deletion(-)

diff --git a/init.sql b/init.sql
index 6cad0a2..76074f9 100644
--- a/init.sql
+++ b/init.sql
@@ -89,7 +89,6 @@ create table uploads (
     size integer not null,
     package_name varchar(256) not null,
     abstract text,
-    stargazers_count integer not null,
 
     version_v integer[] not null,
     version_s varchar(128) not null,

From 454f26187f676da9cac069e184d861b98c50240b Mon Sep 17 00:00:00 2001
From: yuansheng <membphis@gmail.com>
Date: Fri, 28 Oct 2016 09:03:11 +0800
Subject: [PATCH 8/8] 1. 'make update' for updating the postgresql data
 structure 2.  the column 'STARS' shows empty, if there is no value.

---
 Makefile              | 4 ++++
 update.sql            | 2 ++
 web/lua/opmserver.lua | 2 +-
 3 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 update.sql

diff --git a/Makefile b/Makefile
index 51c3f3e..9aa268b 100644
--- a/Makefile
+++ b/Makefile
@@ -68,3 +68,7 @@ clean:
 .PHONY: initdb
 initdb: $(tsv_files)
 	psql -Uopm opm -f init.sql
+
+.PHONY: update
+update: $(tsv_files)
+	psql -Uopm opm -f update.sql
diff --git a/update.sql b/update.sql
new file mode 100644
index 0000000..f274f61
--- /dev/null
+++ b/update.sql
@@ -0,0 +1,2 @@
+
+ALTER TABLE uploads ADD stargazers_count integer;
diff --git a/web/lua/opmserver.lua b/web/lua/opmserver.lua
index 72d483e..f2cfde6 100644
--- a/web/lua/opmserver.lua
+++ b/web/lua/opmserver.lua
@@ -1572,7 +1572,7 @@ do
             local uploader = row.uploader_name
             local org = row.org_name
             local pkg = row.package_name
-            local star = tostring(row.stargazers_count)
+            local star = tostring(row.stargazers_count or "")
 
             local account
             if org and org ~= ngx_null then