handle themes that come from gitlab (instead of from github),
i.e. Midnight https://gitlab.com/BluestNight/Midnight
This commit is contained in:
parent
14e4a84d20
commit
7e4195365a
|
|
@ -13,8 +13,9 @@ from base64 import b64decode
|
|||
from ast import literal_eval
|
||||
|
||||
DATABASENAME='hugothemes.db'
|
||||
# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int);
|
||||
# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int, dot_gitmodules_content text);
|
||||
# CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int);
|
||||
# CREATE TABLE hugothemes_from_gitlab (name varchar unique, url text, commit_sha text, id text, commit_date_in_seconds int, commit_date text, star_count int, themes_toml_content text);
|
||||
THEMESLISTREPO='gohugoio/hugoThemes'
|
||||
|
||||
if len(sys_argv) == 2:
|
||||
|
|
@ -57,9 +58,68 @@ def get_hugo_themes_list():
|
|||
dbconnection.close()
|
||||
|
||||
|
||||
def update_hugo_themes_submodule_list():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select jsondump,dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,))
|
||||
themes_json_string,dot_gitmodule_json_string = cursor.fetchone()
|
||||
themes_json = json_loads(themes_json_string)
|
||||
if dot_gitmodule_json_string:
|
||||
dot_gitmodule_json = json_loads(dot_gitmodule_json_string)
|
||||
for theme in themes_json:
|
||||
if theme['name'] == '.gitmodules':
|
||||
new_gitmodules_sha = theme['sha']
|
||||
break
|
||||
if not dot_gitmodule_json_string or new_gitmodules_sha != dot_gitmodule_json['sha']:
|
||||
api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents/.gitmodules"
|
||||
response = get(api_call_url)
|
||||
if response.status_code == 200:
|
||||
sql = "update hugothemes set dot_gitmodules_content=? where name=?"
|
||||
values=(response.text,THEMESLISTREPO)
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def get_hugo_themes_submodule_list():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,))
|
||||
dot_gitmodule_json_string = cursor.fetchone()[0]
|
||||
dot_gitmodule_json = json_loads(dot_gitmodule_json_string)
|
||||
dot_gitmodule_content = b64decode(dot_gitmodule_json['content']).decode('utf-8').replace('\n\n','\n').split('\n')
|
||||
submodules = []
|
||||
for line in range(len(dot_gitmodule_content[:-1])):
|
||||
if 'submodule' in dot_gitmodule_content[line]:
|
||||
name = re.sub(r'(^\[submodule "|"\]$)','',dot_gitmodule_content[line])
|
||||
if 'path = ' in dot_gitmodule_content[line+1]:
|
||||
path = re.match(r'^.*(path = )(.*)$',dot_gitmodule_content[line+1]).group(2)
|
||||
if 'url = ' in dot_gitmodule_content[line+2]:
|
||||
url = re.match(r'^.*(url = )(.*)$',dot_gitmodule_content[line+2]).group(2)
|
||||
submodules.append((name,path,url))
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
return submodules
|
||||
|
||||
|
||||
def clean_up():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
new_name_list = []
|
||||
submodules = get_hugo_themes_submodule_list()
|
||||
for submodule in submodules:
|
||||
submodule_name = re.sub(r'(^https://github.com/|^https://gitlab.com/|\.git$)','',submodule[2])
|
||||
new_name_list.append(submodule_name)
|
||||
cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,))
|
||||
old_name_list = [name[0] for name in cursor]
|
||||
|
||||
for name in old_name_list:
|
||||
if name not in new_name_list:
|
||||
cursor.execute("delete from hugothemes where name=?",(name,))
|
||||
dbconnection.commit()
|
||||
|
||||
cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,))
|
||||
themes_json_string = cursor.fetchone()[0]
|
||||
themes_json = json_loads(themes_json_string)
|
||||
|
|
@ -70,16 +130,62 @@ def clean_up():
|
|||
split_html_url = theme['html_url'].split('/')
|
||||
new_short_name = split_html_url[3]+'/'+split_html_url[4]
|
||||
new_name_list.append(new_short_name)
|
||||
cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,))
|
||||
old_name_list = cursor.fetchall()
|
||||
|
||||
for name in old_name_list:
|
||||
if name[0] not in new_name_list:
|
||||
cursor.execute("delete from hugothemes where name=?",(name[0],))
|
||||
if name not in new_name_list:
|
||||
cursor.execute("delete from hugothemes where name=?",(name,))
|
||||
dbconnection.commit()
|
||||
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def get_non_github_themes():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name from hugothemes_from_gitlab;")
|
||||
non_github_theme_list = [theme[0] for theme in cursor]
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
return non_github_theme_list
|
||||
|
||||
|
||||
def parse_submodules_from_gitlab():
|
||||
submodules = get_hugo_themes_submodule_list()
|
||||
temp_submodules_list = []
|
||||
for submodule in submodules:
|
||||
if 'gitlab' in submodule[2]:
|
||||
temp_submodules_list.append(submodule)
|
||||
if (len(temp_submodules_list) > 0):
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,))
|
||||
themes_json_string = cursor.fetchone()[0]
|
||||
themes_json = json_loads(themes_json_string)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
submodules_list = []
|
||||
for submodule in temp_submodules_list:
|
||||
for theme in themes_json:
|
||||
if submodule[0] == theme['name']:
|
||||
if (theme['html_url'] == None) or (theme['git_url'] == None):
|
||||
submodules_list.append((submodule[0],submodule[1],submodule[2],theme['sha']))
|
||||
if (len(submodules_list) > 0):
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
for submodule in submodules_list:
|
||||
url = re.sub(r'\.git$','',submodule[2])
|
||||
theme_name = url[19:]
|
||||
cursor.execute("insert or ignore into hugothemes_from_gitlab (name) values (?)",[theme_name])
|
||||
dbconnection.commit()
|
||||
sql = "update hugothemes_from_gitlab set url=?,commit_sha=? where name=?"
|
||||
values=(url,submodule[3],theme_name)
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def parse_hugo_themes_list():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
|
|
@ -101,12 +207,34 @@ def parse_hugo_themes_list():
|
|||
dbconnection.close()
|
||||
|
||||
|
||||
def get_gitlab_project_ids():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,url,id from hugothemes_from_gitlab;")
|
||||
themes = [row for row in cursor]
|
||||
for theme in themes:
|
||||
if theme[2] == None:
|
||||
response = get(theme[1])
|
||||
id_result = re.search(r'(Project ID: )(\d{5,})',response.text).group(2)
|
||||
sql = "update hugothemes_from_gitlab set id=? where name=?"
|
||||
values=(id_result,theme[0])
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
return True
|
||||
|
||||
|
||||
def get_commit_info_for_hugo_themes():
|
||||
non_github_theme_list = get_non_github_themes()
|
||||
themes = []
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,ETag,url,commit_sha from hugothemes where name!=?",(THEMESLISTREPO,))
|
||||
hugo_themes_list = cursor.fetchall()
|
||||
for theme in hugo_themes_list:
|
||||
for theme in cursor:
|
||||
if theme[0] not in non_github_theme_list:
|
||||
themes.append(theme)
|
||||
for theme in themes:
|
||||
name,old_ETag,url,commit_sha = theme
|
||||
api_call_url = 'https://api.github.com/repos/' + name + '/commits/' + commit_sha
|
||||
|
||||
|
|
@ -140,12 +268,46 @@ def get_commit_info_for_hugo_themes():
|
|||
dbconnection.close()
|
||||
|
||||
|
||||
def get_commit_info_for_hugo_themes_from_gitlab():
|
||||
if get_gitlab_project_ids():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,url,commit_sha,id from hugothemes_from_gitlab")
|
||||
hugo_themes_list = cursor.fetchall()
|
||||
for theme in hugo_themes_list:
|
||||
name,url,commit_sha,gitlab_project_id = theme
|
||||
api_call_url = 'https://gitlab.com/api/v4/projects/' + gitlab_project_id + '/repository/commits/' + commit_sha
|
||||
|
||||
response = get(api_call_url)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
commit_date = re.sub(r'(\.\d{3})(Z$)',r'\2',result['created_at'])
|
||||
commit_date_in_seconds = timegm(strptime(commit_date,'%Y-%m-%dT%H:%M:%SZ'))
|
||||
sql = "update hugothemes_from_gitlab set commit_date=?,commit_date_in_seconds=? where name=?"
|
||||
values=(commit_date,commit_date_in_seconds,name)
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
elif response.status_code == 403:
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
write_reports()
|
||||
quit()
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def get_stargazer_count_for_hugo_themes():
|
||||
non_github_theme_list = get_non_github_themes()
|
||||
themes = []
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,repo_ETag from hugothemes where name!=?",(THEMESLISTREPO,))
|
||||
hugo_themes_list = cursor.fetchall()
|
||||
for theme in hugo_themes_list:
|
||||
for theme in cursor:
|
||||
if theme[0] not in non_github_theme_list:
|
||||
themes.append(theme)
|
||||
for theme in themes:
|
||||
name,old_ETag = theme
|
||||
api_call_url = 'https://api.github.com/repos/' + name
|
||||
|
||||
|
|
@ -178,12 +340,44 @@ def get_stargazer_count_for_hugo_themes():
|
|||
dbconnection.close()
|
||||
|
||||
|
||||
def get_stargazer_count_for_hugo_themes_from_gitlab():
|
||||
if get_gitlab_project_ids():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,id from hugothemes_from_gitlab;")
|
||||
themes = cursor.fetchall()
|
||||
for theme in themes:
|
||||
api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1]
|
||||
|
||||
response = get(api_call_url)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
star_count = result['star_count']
|
||||
sql = "update hugothemes_from_gitlab set star_count=? where name=?"
|
||||
values=(star_count,theme[0])
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
elif response.status_code == 403:
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
write_reports()
|
||||
quit()
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def get_theme_dot_toml_for_each_hugo_themes():
|
||||
non_github_theme_list = get_non_github_themes()
|
||||
themes = []
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,))
|
||||
hugo_themes_list = cursor.fetchall()
|
||||
for theme in hugo_themes_list:
|
||||
for theme in cursor:
|
||||
if theme[0] not in non_github_theme_list:
|
||||
themes.append(theme)
|
||||
for theme in themes:
|
||||
name,old_ETag = theme
|
||||
api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml"
|
||||
|
||||
|
|
@ -216,6 +410,51 @@ def get_theme_dot_toml_for_each_hugo_themes():
|
|||
dbconnection.close()
|
||||
|
||||
|
||||
def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
|
||||
if get_gitlab_project_ids():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,id from hugothemes_from_gitlab")
|
||||
themes = cursor.fetchall()
|
||||
for theme in themes:
|
||||
api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1] + '/repository/files/theme.toml?ref=master'
|
||||
|
||||
response = get(api_call_url)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
themes_toml_content = result['content']
|
||||
sql = "update hugothemes_from_gitlab set themes_toml_content=? where name=?"
|
||||
values=(themes_toml_content,theme[0])
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
elif response.status_code == 403:
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
write_reports()
|
||||
quit()
|
||||
print(response.status_code)
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def coalesce_themes():
|
||||
if get_gitlab_project_ids():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
cursor.execute("select name,url,commit_sha,commit_date,commit_date_in_seconds,star_count,themes_toml_content from hugothemes_from_gitlab")
|
||||
themes = cursor.fetchall()
|
||||
for theme in themes:
|
||||
cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme[0]])
|
||||
dbconnection.commit()
|
||||
sql = "update hugothemes set url=?,commit_sha=?,commit_date=?,commit_date_in_seconds=?,stargazers_count=?,themes_toml_content=? where name=?"
|
||||
values=(theme[1],theme[2],theme[3],theme[4],theme[5],theme[6],theme[0])
|
||||
cursor.execute(sql,values)
|
||||
dbconnection.commit()
|
||||
cursor.close()
|
||||
dbconnection.close()
|
||||
|
||||
|
||||
def update_tags_list_for_each_hugo_themes():
|
||||
dbconnection = connect(DATABASENAME)
|
||||
cursor = dbconnection.cursor()
|
||||
|
|
@ -302,7 +541,6 @@ def make_nav_buttons(button_info):
|
|||
return button_block
|
||||
|
||||
|
||||
|
||||
def make_table(themes_info):
|
||||
table = "\t<div class='container'>\n\t\t<table class='table monospace'>\n\t\t\t<thead><tr><th scope='col'>" + themes_info[1] + " (tag) " + themes_info[2] + "</th><th scope='col'>Date</th></tr></thead>\n\t\t\t<tbody>\n"
|
||||
for theme in themes_info[0]:
|
||||
|
|
@ -367,12 +605,19 @@ def write_reports():
|
|||
|
||||
if __name__=="__main__":
|
||||
get_hugo_themes_list()
|
||||
update_hugo_themes_submodule_list()
|
||||
clean_up()
|
||||
parse_submodules_from_gitlab()
|
||||
parse_hugo_themes_list()
|
||||
get_commit_info_for_hugo_themes()
|
||||
get_commit_info_for_hugo_themes_from_gitlab()
|
||||
get_stargazer_count_for_hugo_themes()
|
||||
get_stargazer_count_for_hugo_themes_from_gitlab()
|
||||
get_theme_dot_toml_for_each_hugo_themes()
|
||||
get_theme_dot_toml_for_each_hugo_themes_from_gitlab()
|
||||
coalesce_themes()
|
||||
update_tags_list_for_each_hugo_themes()
|
||||
update_tag_table()
|
||||
write_reports()
|
||||
"""
|
||||
"""
|
||||
|
|
|
|||
Loading…
Reference in New Issue