From 7e4195365a6b9de90799b75d957ae48a5877d08e Mon Sep 17 00:00:00 2001 From: Trent Palmer Date: Mon, 24 Dec 2018 04:30:18 -0800 Subject: [PATCH] handle themes that come from gitlab (instead of from github), i.e. Midnight https://gitlab.com/BluestNight/Midnight --- rank_hugo_themes.py | 269 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 257 insertions(+), 12 deletions(-) diff --git a/rank_hugo_themes.py b/rank_hugo_themes.py index 451af95..84be302 100755 --- a/rank_hugo_themes.py +++ b/rank_hugo_themes.py @@ -13,8 +13,9 @@ from base64 import b64decode from ast import literal_eval DATABASENAME='hugothemes.db' -# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int); +# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int, dot_gitmodules_content text); # CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int); +# CREATE TABLE hugothemes_from_gitlab (name varchar unique, url text, commit_sha text, id text, commit_date_in_seconds int, commit_date text, star_count int, themes_toml_content text); THEMESLISTREPO='gohugoio/hugoThemes' if len(sys_argv) == 2: @@ -57,9 +58,68 @@ def get_hugo_themes_list(): dbconnection.close() +def update_hugo_themes_submodule_list(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select jsondump,dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,)) + themes_json_string,dot_gitmodule_json_string = cursor.fetchone() + themes_json = json_loads(themes_json_string) + if dot_gitmodule_json_string: + dot_gitmodule_json = json_loads(dot_gitmodule_json_string) + for theme in themes_json: + if theme['name'] == '.gitmodules': + new_gitmodules_sha = theme['sha'] + break + if not dot_gitmodule_json_string or new_gitmodules_sha != dot_gitmodule_json['sha']: + api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents/.gitmodules" + response = get(api_call_url) + if response.status_code == 200: + sql = "update hugothemes set dot_gitmodules_content=? where name=?" + values=(response.text,THEMESLISTREPO) + cursor.execute(sql,values) + dbconnection.commit() + print(response.status_code) + cursor.close() + dbconnection.close() + + +def get_hugo_themes_submodule_list(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,)) + dot_gitmodule_json_string = cursor.fetchone()[0] + dot_gitmodule_json = json_loads(dot_gitmodule_json_string) + dot_gitmodule_content = b64decode(dot_gitmodule_json['content']).decode('utf-8').replace('\n\n','\n').split('\n') + submodules = [] + for line in range(len(dot_gitmodule_content[:-1])): + if 'submodule' in dot_gitmodule_content[line]: + name = re.sub(r'(^\[submodule "|"\]$)','',dot_gitmodule_content[line]) + if 'path = ' in dot_gitmodule_content[line+1]: + path = re.match(r'^.*(path = )(.*)$',dot_gitmodule_content[line+1]).group(2) + if 'url = ' in dot_gitmodule_content[line+2]: + url = re.match(r'^.*(url = )(.*)$',dot_gitmodule_content[line+2]).group(2) + submodules.append((name,path,url)) + cursor.close() + dbconnection.close() + return submodules + + def clean_up(): dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() + new_name_list = [] + submodules = get_hugo_themes_submodule_list() + for submodule in submodules: + submodule_name = re.sub(r'(^https://github.com/|^https://gitlab.com/|\.git$)','',submodule[2]) + new_name_list.append(submodule_name) + cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,)) + old_name_list = [name[0] for name in cursor] + + for name in old_name_list: + if name not in new_name_list: + cursor.execute("delete from hugothemes where name=?",(name,)) + dbconnection.commit() + cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) themes_json_string = cursor.fetchone()[0] themes_json = json_loads(themes_json_string) @@ -70,16 +130,62 @@ def clean_up(): split_html_url = theme['html_url'].split('/') new_short_name = split_html_url[3]+'/'+split_html_url[4] new_name_list.append(new_short_name) - cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,)) - old_name_list = cursor.fetchall() + for name in old_name_list: - if name[0] not in new_name_list: - cursor.execute("delete from hugothemes where name=?",(name[0],)) + if name not in new_name_list: + cursor.execute("delete from hugothemes where name=?",(name,)) dbconnection.commit() + cursor.close() dbconnection.close() +def get_non_github_themes(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name from hugothemes_from_gitlab;") + non_github_theme_list = [theme[0] for theme in cursor] + cursor.close() + dbconnection.close() + return non_github_theme_list + + +def parse_submodules_from_gitlab(): + submodules = get_hugo_themes_submodule_list() + temp_submodules_list = [] + for submodule in submodules: + if 'gitlab' in submodule[2]: + temp_submodules_list.append(submodule) + if (len(temp_submodules_list) > 0): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) + themes_json_string = cursor.fetchone()[0] + themes_json = json_loads(themes_json_string) + cursor.close() + dbconnection.close() + submodules_list = [] + for submodule in temp_submodules_list: + for theme in themes_json: + if submodule[0] == theme['name']: + if (theme['html_url'] == None) or (theme['git_url'] == None): + submodules_list.append((submodule[0],submodule[1],submodule[2],theme['sha'])) + if (len(submodules_list) > 0): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + for submodule in submodules_list: + url = re.sub(r'\.git$','',submodule[2]) + theme_name = url[19:] + cursor.execute("insert or ignore into hugothemes_from_gitlab (name) values (?)",[theme_name]) + dbconnection.commit() + sql = "update hugothemes_from_gitlab set url=?,commit_sha=? where name=?" + values=(url,submodule[3],theme_name) + cursor.execute(sql,values) + dbconnection.commit() + cursor.close() + dbconnection.close() + + def parse_hugo_themes_list(): dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() @@ -101,12 +207,34 @@ def parse_hugo_themes_list(): dbconnection.close() +def get_gitlab_project_ids(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,url,id from hugothemes_from_gitlab;") + themes = [row for row in cursor] + for theme in themes: + if theme[2] == None: + response = get(theme[1]) + id_result = re.search(r'(Project ID: )(\d{5,})',response.text).group(2) + sql = "update hugothemes_from_gitlab set id=? where name=?" + values=(id_result,theme[0]) + cursor.execute(sql,values) + dbconnection.commit() + cursor.close() + dbconnection.close() + return True + + def get_commit_info_for_hugo_themes(): + non_github_theme_list = get_non_github_themes() + themes = [] dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() cursor.execute("select name,ETag,url,commit_sha from hugothemes where name!=?",(THEMESLISTREPO,)) - hugo_themes_list = cursor.fetchall() - for theme in hugo_themes_list: + for theme in cursor: + if theme[0] not in non_github_theme_list: + themes.append(theme) + for theme in themes: name,old_ETag,url,commit_sha = theme api_call_url = 'https://api.github.com/repos/' + name + '/commits/' + commit_sha @@ -140,12 +268,46 @@ def get_commit_info_for_hugo_themes(): dbconnection.close() +def get_commit_info_for_hugo_themes_from_gitlab(): + if get_gitlab_project_ids(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,url,commit_sha,id from hugothemes_from_gitlab") + hugo_themes_list = cursor.fetchall() + for theme in hugo_themes_list: + name,url,commit_sha,gitlab_project_id = theme + api_call_url = 'https://gitlab.com/api/v4/projects/' + gitlab_project_id + '/repository/commits/' + commit_sha + + response = get(api_call_url) + if response.status_code == 200: + result = response.json() + commit_date = re.sub(r'(\.\d{3})(Z$)',r'\2',result['created_at']) + commit_date_in_seconds = timegm(strptime(commit_date,'%Y-%m-%dT%H:%M:%SZ')) + sql = "update hugothemes_from_gitlab set commit_date=?,commit_date_in_seconds=? where name=?" + values=(commit_date,commit_date_in_seconds,name) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + + def get_stargazer_count_for_hugo_themes(): + non_github_theme_list = get_non_github_themes() + themes = [] dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() cursor.execute("select name,repo_ETag from hugothemes where name!=?",(THEMESLISTREPO,)) - hugo_themes_list = cursor.fetchall() - for theme in hugo_themes_list: + for theme in cursor: + if theme[0] not in non_github_theme_list: + themes.append(theme) + for theme in themes: name,old_ETag = theme api_call_url = 'https://api.github.com/repos/' + name @@ -178,12 +340,44 @@ def get_stargazer_count_for_hugo_themes(): dbconnection.close() +def get_stargazer_count_for_hugo_themes_from_gitlab(): + if get_gitlab_project_ids(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,id from hugothemes_from_gitlab;") + themes = cursor.fetchall() + for theme in themes: + api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1] + + response = get(api_call_url) + if response.status_code == 200: + result = response.json() + star_count = result['star_count'] + sql = "update hugothemes_from_gitlab set star_count=? where name=?" + values=(star_count,theme[0]) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + + def get_theme_dot_toml_for_each_hugo_themes(): + non_github_theme_list = get_non_github_themes() + themes = [] dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,)) - hugo_themes_list = cursor.fetchall() - for theme in hugo_themes_list: + for theme in cursor: + if theme[0] not in non_github_theme_list: + themes.append(theme) + for theme in themes: name,old_ETag = theme api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml" @@ -216,6 +410,51 @@ def get_theme_dot_toml_for_each_hugo_themes(): dbconnection.close() +def get_theme_dot_toml_for_each_hugo_themes_from_gitlab(): + if get_gitlab_project_ids(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,id from hugothemes_from_gitlab") + themes = cursor.fetchall() + for theme in themes: + api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1] + '/repository/files/theme.toml?ref=master' + + response = get(api_call_url) + if response.status_code == 200: + result = response.json() + themes_toml_content = result['content'] + sql = "update hugothemes_from_gitlab set themes_toml_content=? where name=?" + values=(themes_toml_content,theme[0]) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + + +def coalesce_themes(): + if get_gitlab_project_ids(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,url,commit_sha,commit_date,commit_date_in_seconds,star_count,themes_toml_content from hugothemes_from_gitlab") + themes = cursor.fetchall() + for theme in themes: + cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme[0]]) + dbconnection.commit() + sql = "update hugothemes set url=?,commit_sha=?,commit_date=?,commit_date_in_seconds=?,stargazers_count=?,themes_toml_content=? where name=?" + values=(theme[1],theme[2],theme[3],theme[4],theme[5],theme[6],theme[0]) + cursor.execute(sql,values) + dbconnection.commit() + cursor.close() + dbconnection.close() + + def update_tags_list_for_each_hugo_themes(): dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() @@ -302,7 +541,6 @@ def make_nav_buttons(button_info): return button_block - def make_table(themes_info): table = "\t
\n\t\t\n\t\t\t\n\t\t\t\n" for theme in themes_info[0]: @@ -367,12 +605,19 @@ def write_reports(): if __name__=="__main__": get_hugo_themes_list() + update_hugo_themes_submodule_list() clean_up() + parse_submodules_from_gitlab() parse_hugo_themes_list() get_commit_info_for_hugo_themes() + get_commit_info_for_hugo_themes_from_gitlab() get_stargazer_count_for_hugo_themes() + get_stargazer_count_for_hugo_themes_from_gitlab() get_theme_dot_toml_for_each_hugo_themes() + get_theme_dot_toml_for_each_hugo_themes_from_gitlab() + coalesce_themes() update_tags_list_for_each_hugo_themes() + update_tag_table() write_reports() """ """
" + themes_info[1] + " (tag) " + themes_info[2] + "Date