From 33e783964f1b92e83f93ed44761b54f11b8afe35 Mon Sep 17 00:00:00 2001 From: Trent Palmer Date: Wed, 19 Dec 2018 22:05:08 -0800 Subject: [PATCH] combine two report pages into one, add sort by tag to report --- rank_hugo_themes.py | 260 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 204 insertions(+), 56 deletions(-) diff --git a/rank_hugo_themes.py b/rank_hugo_themes.py index 7b92469..451af95 100755 --- a/rank_hugo_themes.py +++ b/rank_hugo_themes.py @@ -1,16 +1,20 @@ -#!/usr/bin/env python36 +#!/usr/bin/env python3 # rank_hugo_themes.py -# import sqlite3,json,sys,calendar,time,requests +import re +import toml from calendar import timegm from time import strptime from requests import get from sqlite3 import connect from json import loads as json_loads from sys import argv as sys_argv +from base64 import b64decode +from ast import literal_eval DATABASENAME='hugothemes.db' -# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int); +# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int); +# CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int); THEMESLISTREPO='gohugoio/hugoThemes' if len(sys_argv) == 2: @@ -61,10 +65,11 @@ def clean_up(): themes_json = json_loads(themes_json_string) new_name_list = [] for theme in themes_json: - if 'gohugoio' not in theme['git_url']: - split_html_url = theme['html_url'].split('/') - new_short_name = split_html_url[3]+'/'+split_html_url[4] - new_name_list.append(new_short_name) + if theme['git_url']: + if 'gohugoio' not in theme['git_url']: + split_html_url = theme['html_url'].split('/') + new_short_name = split_html_url[3]+'/'+split_html_url[4] + new_name_list.append(new_short_name) cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,)) old_name_list = cursor.fetchall() for name in old_name_list: @@ -82,15 +87,16 @@ def parse_hugo_themes_list(): themes_json_string = cursor.fetchone()[0] themes_json = json_loads(themes_json_string) for x in themes_json: - if 'gohugoio' not in x['git_url']: - theme_git_url = x['html_url'][:-46] - theme_git_name = theme_git_url[19:] - cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name]) - dbconnection.commit() - sql = "update hugothemes set url=?,commit_sha=? where name=?" - values=(theme_git_url,x['sha'],theme_git_name) - cursor.execute(sql,values) - dbconnection.commit() + if x['git_url']: + if 'gohugoio' not in x['git_url']: + theme_git_url = x['html_url'][:-46] + theme_git_name = theme_git_url[19:] + cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name]) + dbconnection.commit() + sql = "update hugothemes set url=?,commit_sha=? where name=?" + values=(theme_git_url,x['sha'],theme_git_name) + cursor.execute(sql,values) + dbconnection.commit() cursor.close() dbconnection.close() @@ -171,53 +177,191 @@ def get_stargazer_count_for_hugo_themes(): cursor.close() dbconnection.close() + +def get_theme_dot_toml_for_each_hugo_themes(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + for theme in hugo_themes_list: + name,old_ETag = theme + api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml" + + if old_ETag != None: + headers['If-None-Match'] = old_ETag + else: + if 'If-None-Match' in headers: + del headers['If-None-Match'] + + if len(headers) == 0: + response = get(api_call_url) + else: + response = get(api_call_url, headers=headers) + if response.status_code == 200: + themes_toml_ETag = response.headers['ETag'] + result = response.json() + themes_toml_content = result['content'] + sql = "update hugothemes set themes_toml_ETag=?,themes_toml_content=? where name=?" + values=(themes_toml_ETag,themes_toml_content,name) + cursor.execute(sql,values) + dbconnection.commit() + elif response.status_code == 403: + print(response.status_code) + cursor.close() + dbconnection.close() + write_reports() + quit() + print(response.status_code) + cursor.close() + dbconnection.close() + + +def update_tags_list_for_each_hugo_themes(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,themes_toml_content from hugothemes where name!=?",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + for theme in hugo_themes_list: + if theme[1] != None: + content = b64decode(theme[1]).decode('utf-8') + # put quotes around any unquoted double-dotted version numbers + # because python toml libraries will error out on those + theme_toml = toml.loads(re.sub(r'\s(\d+\.\d+\.\d+)\s',r'"\1"',content)) + if 'tags' in theme_toml: + if len(theme_toml['tags']) > 0: + theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0] + num_tags = len(theme_tags) + if num_tags > 0: + tag_list = str(theme_tags) + else: + tag_list = None + else: + tag_list = None + num_tags = 0 + else: + tag_list = None + num_tags = 0 + else: + tag_list = None + num_tags = 0 + sql = "update hugothemes set tags_list=?,num_tags=? where name=?" + values=(tag_list,num_tags,theme[0]) + cursor.execute(sql,values) + dbconnection.commit() + cursor.close() + dbconnection.close() + + +def update_tag_table(): + dbconnection = connect(DATABASENAME) + cursor = dbconnection.cursor() + cursor.execute("select name,tags_list from hugothemes where name!=?",(THEMESLISTREPO,)) + hugo_themes_list = cursor.fetchall() + tags_list = [] + for theme in hugo_themes_list: + if theme[1] != None: + tags = literal_eval(theme[1]) + for tag in tags: + if len(tag) > 0: + if tag not in tags_list: + tags_list.append(tag) + + for tag in tags_list: + cursor.execute("insert or ignore into tags (tag) values (?)",[tag]) + dbconnection.commit() + theme_list = [] + for theme in hugo_themes_list: + if theme[1] != None: + tags = literal_eval(theme[1]) + if tag in tags: + theme_list.append(theme[0]) + sql = "update tags set theme_list=?,num_themes=? where tag=?" + values=(str(theme_list),len(theme_list),tag) + cursor.execute(sql,values) + dbconnection.commit() + + cursor.close() + dbconnection.close() + + +def make_buttons(tags_list): + button_block = "\t\t
\n" + for tag in tags_list: + button_block += "\t\t\t
\n" + button_block += "\t\t
\n" + return button_block + + +def make_nav_buttons(button_info): + button_block = "\t\t
\n" + button_block += "\t\t\t
\n" + if button_info[2] > 10: + button_block += "\t\t\t
\n" + button_block += "\t\t\t
\n" + button_block += "\t\t
\n" + return button_block + + + +def make_table(themes_info): + table = "\t
\n\t\t\n\t\t\t\n\t\t\t\n" + for theme in themes_info[0]: + name = theme[0].split('/')[1] + row = "\t\t\t\t" + row += "" + row += "" + row += "\n" + table += row + table += "\t\t\t\n\t\t
" + themes_info[1] + " (tag) " + themes_info[2] + "Date
"+""+name+""""+theme[2][:10]+""+str(theme[3])+'★'+""+theme[1][:6]+"
\n\t
\n" + return table + + +def make_section(section_info): + if section_info[3] > 10: + section = make_nav_buttons((section_info[0],'by-date',section_info[3])) + section += make_table((section_info[1],section_info[0],'by-date')) + section += make_nav_buttons((section_info[0],'by-stars',section_info[3])) + section += make_table((section_info[2],section_info[0],'by-stars')) + else: + section = make_nav_buttons((section_info[0],'by-date',section_info[3])) + section += make_table((section_info[1],section_info[0],'by-date')) + return section + + def write_reports(): dbconnection = connect(DATABASENAME) cursor = dbconnection.cursor() cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by commit_date_in_seconds DESC",(THEMESLISTREPO,)) - hugo_themes_list = cursor.fetchall() - cursor.close() - dbconnection.close() - bycommitTable = "\n\n" - bycommitTable += "\t\n\t\tHugo Themes Report Order By Commit Date\n\t\t" - bycommitTable += "\n\t\t\n" - bycommitTable += "\t\t\n\t\n" - bycommitTable += "\t
(Switch-to-Order-by-Num-Stars)
\n" - bycommitTable +="\t
\n\t\t\n\t\t\t\n\t\t\t\n" - for theme in hugo_themes_list: - name = theme[0].split('/')[1] - row = "\t\t\t\t" - row += "" - row += "" - row += "\n" - bycommitTable += row - bycommitTable += "\t\t\t\n\t\t
NameDate
"+""+name+""""+theme[2][:10]+""+str(theme[3])+'★'+""+theme[1][:6]+"
\n\t
\n" - by_date = open('hugo-themes-report/hugo-themes-by-last-commit-date.html','w') - by_date.write(bycommitTable) - by_date.close() - - dbconnection = connect(DATABASENAME) - cursor = dbconnection.cursor() + themes_bydate_list = cursor.fetchall() cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by stargazers_count DESC",(THEMESLISTREPO,)) - hugo_themes_list = cursor.fetchall() + themes_bystars_list = cursor.fetchall() + themes = [theme[0] for theme in themes_bystars_list] + tags_list = [('all',str(themes),len(themes))] + cursor.execute("select tag,theme_list,num_themes from tags where num_themes > 2 order by num_themes DESC") + for row in cursor: + tags_list.append(row) cursor.close() dbconnection.close() - bystarsTable = "\n\n" - bystarsTable += "\t\n\t\tHugo Themes Report Order By Num Stars\n\t\t" - bystarsTable += "\n\t\t\n" - bystarsTable += "\t\t\n\t\n" - bystarsTable += "\t
(Switch-Order-by-Commit-Date)
\n" - bystarsTable += "\t
\n\t\t\n\t\t\t\n\t\t\t\n" - for theme in hugo_themes_list: - name = theme[0].split('/')[1] - row = "\t\t\t\t" - row += "" - row += "" - row += "\n" - bystarsTable += row - bystarsTable += "\t\t\t\n\t\t
NameDate
"+""+name+""""+theme[2][:10]+""+str(theme[3])+'★'+""+theme[1][:6]+"
\n\t
\n" - by_date = open('hugo-themes-report/hugo-themes-by-num-stars.html','w') - by_date.write(bystarsTable) + + reportpage = "\n\n" + reportpage += "\t\n\t\tHugo Themes Report\n\t\t" + reportpage += "\n\t\t\n" + reportpage += "\t\t