diff --git a/rank_hugo_themes.py b/rank_hugo_themes.py new file mode 100755 index 0000000..7b92469 --- /dev/null +++ b/rank_hugo_themes.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python36 +# rank_hugo_themes.py + +# import sqlite3,json,sys,calendar,time,requests +from calendar import timegm +from time import strptime +from requests import get +from sqlite3 import connect +from json import loads as json_loads +from sys import argv as sys_argv + +DATABASENAME='hugothemes.db' +# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int); +THEMESLISTREPO='gohugoio/hugoThemes' + +if len(sys_argv) == 2: + headers = { 'Authorization' : 'token '+sys_argv[1] } +else: + headers = {} + + +def get_hugo_themes_list(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select ETag from hugothemes where name=?",(THEMESLISTREPO,)) + old_ETag = cursor.fetchone()[0] + api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents" + + if old_ETag != None: + headers['If-None-Match'] = old_ETag + else: + if 'If-None-Match' in headers: + del headers['If-None-Match'] + + if len(headers) == 0: + response = get(api_call_url) + else: + response = get(api_call_url, headers=headers) + if response.status_code == 200: + ETag = response.headers['ETag'] + sql = "update hugothemes set ETag=?,url=?,jsondump=? where name=?" + values=(ETag,api_call_url,response.text,THEMESLISTREPO) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + + +def clean_up(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) + themes_json_string = cursor.fetchone()[0] + themes_json = json_loads(themes_json_string) + new_name_list = [] + for theme in themes_json: + if 'gohugoio' not in theme['git_url']: + split_html_url = theme['html_url'].split('/') + new_short_name = split_html_url[3]+'/'+split_html_url[4] + new_name_list.append(new_short_name) + cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,)) + old_name_list = cursor.fetchall() + for name in old_name_list: + if name[0] not in new_name_list: + cursor.execute("delete from hugothemes where name=?",(name[0],)) + dbconnection.commit() + cursor.close() + dbconnection.close() + + +def parse_hugo_themes_list(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) + themes_json_string = cursor.fetchone()[0] + themes_json = json_loads(themes_json_string) + for x in themes_json: + if 'gohugoio' not in x['git_url']: + theme_git_url = x['html_url'][:-46] + theme_git_name = theme_git_url[19:] + cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name]) + dbconnection.commit() + sql = "update hugothemes set url=?,commit_sha=? where name=?" + values=(theme_git_url,x['sha'],theme_git_name) + cursor.execute(sql,values) + dbconnection.commit() + cursor.close() + dbconnection.close() + + +def get_commit_info_for_hugo_themes(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,ETag,url,commit_sha from hugothemes where name!=?",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + for theme in hugo_themes_list: + name,old_ETag,url,commit_sha = theme + api_call_url = 'https://api.github.com/repos/' + name + '/commits/' + commit_sha + + if old_ETag != None: + headers['If-None-Match'] = old_ETag + else: + if 'If-None-Match' in headers: + del headers['If-None-Match'] + + if len(headers) == 0: + response = get(api_call_url) + else: + response = get(api_call_url, headers=headers) + if response.status_code == 200: + ETag = response.headers['ETag'] + result = response.json() + commit_date = result['commit']['author']['date'] + commit_date_in_seconds = timegm(strptime(commit_date,'%Y-%m-%dT%H:%M:%SZ')) + sql = "update hugothemes set ETag=?,commit_date=?,commit_date_in_seconds=? where name=?" + values=(ETag,commit_date,commit_date_in_seconds,name) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + + +def get_stargazer_count_for_hugo_themes(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,repo_ETag from hugothemes where name!=?",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + for theme in hugo_themes_list: + name,old_ETag = theme + api_call_url = 'https://api.github.com/repos/' + name + + if old_ETag != None: + headers['If-None-Match'] = old_ETag + else: + if 'If-None-Match' in headers: + del headers['If-None-Match'] + + if len(headers) == 0: + response = get(api_call_url) + else: + response = get(api_call_url, headers=headers) + if response.status_code == 200: + repo_ETag = response.headers['ETag'] + result = response.json() + stargazers_count = result['stargazers_count'] + sql = "update hugothemes set repo_ETag=?,stargazers_count=? where name=?" + values=(repo_ETag,stargazers_count,name) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + +def write_reports(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by commit_date_in_seconds DESC",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + cursor.close() + dbconnection.close() + bycommitTable = "\n\n" + bycommitTable += "\t\n\t\tHugo Themes Report Order By Commit Date\n\t\t" + bycommitTable += "\n\t\t\n" + bycommitTable += "\t\t\n\t\n" + bycommitTable += "\t
(Switch-to-Order-by-Num-Stars)
\n" + bycommitTable +="\t
\n\t\t\n\t\t\t\n\t\t\t\n" + for theme in hugo_themes_list: + name = theme[0].split('/')[1] + row = "\t\t\t\t" + row += "" + row += "" + row += "\n" + bycommitTable += row + bycommitTable += "\t\t\t\n\t\t
NameDate
"+""+name+""""+theme[2][:10]+""+str(theme[3])+'★'+""+theme[1][:6]+"
\n\t
\n" + by_date = open('hugo-themes-report/hugo-themes-by-last-commit-date.html','w') + by_date.write(bycommitTable) + by_date.close() + + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by stargazers_count DESC",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + cursor.close() + dbconnection.close() + bystarsTable = "\n\n" + bystarsTable += "\t\n\t\tHugo Themes Report Order By Num Stars\n\t\t" + bystarsTable += "\n\t\t\n" + bystarsTable += "\t\t\n\t\n" + bystarsTable += "\t
(Switch-Order-by-Commit-Date)
\n" + bystarsTable += "\t
\n\t\t\n\t\t\t\n\t\t\t\n" + for theme in hugo_themes_list: + name = theme[0].split('/')[1] + row = "\t\t\t\t" + row += "" + row += "" + row += "\n" + bystarsTable += row + bystarsTable += "\t\t\t\n\t\t
NameDate
"+""+name+""""+theme[2][:10]+""+str(theme[3])+'★'+""+theme[1][:6]+"
\n\t
\n" + by_date = open('hugo-themes-report/hugo-themes-by-num-stars.html','w') + by_date.write(bystarsTable) + by_date.close() + + +if __name__=="__main__": + get_hugo_themes_list() + clean_up() + parse_hugo_themes_list() + get_commit_info_for_hugo_themes() + get_stargazer_count_for_hugo_themes() + write_reports()