trent/hugo_themes_report
1
0
Fork 0
mirror of https://github.com/TrentSPalmer/hugo_themes_report.git synced 2024-12-04 16:41:31 -08:00
Code Issues Projects Releases Wiki Activity

combine two report pages into one, add sort by tag to report

Browse Source
This commit is contained in:
Trent Palmer 2018-12-19 22:05:08 -08:00
parent 1f8cfdb021
commit 33e783964f
1 changed files with 204 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

260
rank_hugo_themes.py
View File

@ -1,16 +1,20 @@
#!/usr/bin/env python36
#!/usr/bin/env python3
# rank_hugo_themes.py
# import sqlite3,json,sys,calendar,time,requests
import re
import toml
from calendar import timegm
from time import strptime
from requests import get
from sqlite3 import connect
from json import loads as json_loads
from sys import argv as sys_argv
from base64 import b64decode
from ast import literal_eval
DATABASENAME='hugothemes.db'
# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int);
# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int);
# CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int);
THEMESLISTREPO='gohugoio/hugoThemes'
if len(sys_argv) == 2:
@ -61,10 +65,11 @@ def clean_up():
themes_json = json_loads(themes_json_string)
new_name_list = []
for theme in themes_json:
if 'gohugoio' not in theme['git_url']:
split_html_url = theme['html_url'].split('/')
new_short_name = split_html_url[3]+'/'+split_html_url[4]
new_name_list.append(new_short_name)
if theme['git_url']:
if 'gohugoio' not in theme['git_url']:
split_html_url = theme['html_url'].split('/')
new_short_name = split_html_url[3]+'/'+split_html_url[4]
new_name_list.append(new_short_name)
cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,))
old_name_list = cursor.fetchall()
for name in old_name_list:
@ -82,15 +87,16 @@ def parse_hugo_themes_list():
themes_json_string = cursor.fetchone()[0]
themes_json = json_loads(themes_json_string)
for x in themes_json:
if 'gohugoio' not in x['git_url']:
theme_git_url = x['html_url'][:-46]
theme_git_name = theme_git_url[19:]
cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name])
dbconnection.commit()
sql = "update hugothemes set url=?,commit_sha=? where name=?"
values=(theme_git_url,x['sha'],theme_git_name)
cursor.execute(sql,values)
dbconnection.commit()
if x['git_url']:
if 'gohugoio' not in x['git_url']:
theme_git_url = x['html_url'][:-46]
theme_git_name = theme_git_url[19:]
cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name])
dbconnection.commit()
sql = "update hugothemes set url=?,commit_sha=? where name=?"
values=(theme_git_url,x['sha'],theme_git_name)
cursor.execute(sql,values)
dbconnection.commit()
cursor.close()
dbconnection.close()
@ -171,53 +177,191 @@ def get_stargazer_count_for_hugo_themes():
cursor.close()
dbconnection.close()
def get_theme_dot_toml_for_each_hugo_themes():
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,))
hugo_themes_list = cursor.fetchall()
for theme in hugo_themes_list:
name,old_ETag = theme
api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml"
if old_ETag != None:
headers['If-None-Match'] = old_ETag
else:
if 'If-None-Match' in headers:
del headers['If-None-Match']
if len(headers) == 0:
response = get(api_call_url)
else:
response = get(api_call_url, headers=headers)
if response.status_code == 200:
themes_toml_ETag = response.headers['ETag']
result = response.json()
themes_toml_content = result['content']
sql = "update hugothemes set themes_toml_ETag=?,themes_toml_content=? where name=?"
values=(themes_toml_ETag,themes_toml_content,name)
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403:
print(response.status_code)
cursor.close()
dbconnection.close()
write_reports()
quit()
print(response.status_code)
cursor.close()
dbconnection.close()
def update_tags_list_for_each_hugo_themes():
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
cursor.execute("select name,themes_toml_content from hugothemes where name!=?",(THEMESLISTREPO,))
hugo_themes_list = cursor.fetchall()
for theme in hugo_themes_list:
if theme[1] != None:
content = b64decode(theme[1]).decode('utf-8')
# put quotes around any unquoted double-dotted version numbers
# because python toml libraries will error out on those
theme_toml = toml.loads(re.sub(r'\s(\d+\.\d+\.\d+)\s',r'"\1"',content))
if 'tags' in theme_toml:
if len(theme_toml['tags']) > 0:
theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0]
num_tags = len(theme_tags)
if num_tags > 0:
tag_list = str(theme_tags)
else:
tag_list = None
else:
tag_list = None
num_tags = 0
else:
tag_list = None
num_tags = 0
else:
tag_list = None
num_tags = 0
sql = "update hugothemes set tags_list=?,num_tags=? where name=?"
values=(tag_list,num_tags,theme[0])
cursor.execute(sql,values)
dbconnection.commit()
cursor.close()
dbconnection.close()
def update_tag_table():
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
cursor.execute("select name,tags_list from hugothemes where name!=?",(THEMESLISTREPO,))
hugo_themes_list = cursor.fetchall()
tags_list = []
for theme in hugo_themes_list:
if theme[1] != None:
tags = literal_eval(theme[1])
for tag in tags:
if len(tag) > 0:
if tag not in tags_list:
tags_list.append(tag)
for tag in tags_list:
cursor.execute("insert or ignore into tags (tag) values (?)",[tag])
dbconnection.commit()
theme_list = []
for theme in hugo_themes_list:
if theme[1] != None:
tags = literal_eval(theme[1])
if tag in tags:
theme_list.append(theme[0])
sql = "update tags set theme_list=?,num_themes=? where tag=?"
values=(str(theme_list),len(theme_list),tag)
cursor.execute(sql,values)
dbconnection.commit()
cursor.close()
dbconnection.close()
def make_buttons(tags_list):
button_block = "\t\t<div id='all-tags' class='d-flex flex-wrap justify-content-around'>\n"
for tag in tags_list:
button_block += "\t\t\t<div class='d-flex'><a href='#" + tag[0] + "-by-date'><button type='button' class='btn btn-outline-primary'>" + tag[0] + ", " + str(tag[2]) + "</button></a></div>\n"
button_block += "\t\t</div>\n"
return button_block
def make_nav_buttons(button_info):
button_block = "\t\t<div id='" + button_info[0] + "-" + button_info[1] + "' class='d-flex flex-wrap justify-content-around'>\n"
button_block += "\t\t\t<div class='d-flex'><a href='#all-tags'><button type='button' class='btn btn-outline-primary'>tags</button></a></div>\n"
if button_info[2] > 10:
button_block += "\t\t\t<div class='d-flex'><a href='#" + button_info[0] + "-by-date'><button type='button' class='btn btn-outline-primary'>" + button_info[0] + " by date</button></a></div>\n"
button_block += "\t\t\t<div class='d-flex'><a href='#" + button_info[0] + "-by-stars'><button type='button' class='btn btn-outline-primary'>" + button_info[0] + " by stars</button></a></div>\n"
button_block += "\t\t</div>\n"
return button_block
def make_table(themes_info):
table = "\t<div class='container'>\n\t\t<table class='table monospace'>\n\t\t\t<thead><tr><th scope='col'>" + themes_info[1] + " (tag) " + themes_info[2] + "</th><th scope='col'>Date</th></tr></thead>\n\t\t\t<tbody>\n"
for theme in themes_info[0]:
name = theme[0].split('/')[1]
row = "\t\t\t\t<tr><td scope='row'>"+"<a href="+theme[4]+">"+name+"</a>""</td>"
row += "<td nowrap>"+theme[2][:10]+"</td>"
row += "<td align='right' nowrap style='padding-left:1em;'>"+str(theme[3])+'&#x2605'+"</td>"
row += "<td align='right' style='padding-left:1em;'>"+theme[1][:6]+"</td></tr>\n"
table += row
table += "\t\t\t</tbody>\n\t\t</table>\n\t</div>\n"
return table
def make_section(section_info):
if section_info[3] > 10:
section = make_nav_buttons((section_info[0],'by-date',section_info[3]))
section += make_table((section_info[1],section_info[0],'by-date'))
section += make_nav_buttons((section_info[0],'by-stars',section_info[3]))
section += make_table((section_info[2],section_info[0],'by-stars'))
else:
section = make_nav_buttons((section_info[0],'by-date',section_info[3]))
section += make_table((section_info[1],section_info[0],'by-date'))
return section
def write_reports():
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by commit_date_in_seconds DESC",(THEMESLISTREPO,))
hugo_themes_list = cursor.fetchall()
cursor.close()
dbconnection.close()
bycommitTable = "<!DOCTYPE html>\n<html lang='en'>\n"
bycommitTable += "\t<head>\n\t\t<title>Hugo Themes Report Order By Commit Date</title>\n\t\t<meta charset='utf-8'>"
bycommitTable += "\n\t\t<meta name='viewport' content='width=device-width, initial-scale=1'>\n"
bycommitTable += "\t\t<link rel='stylesheet' href='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css' integrity='sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u' crossorigin='anonymous'>\n\t</head>\n"
bycommitTable += "\t<div style='text-align:center;font-size:200%;'><a style='text-decoration:none;'href=\"hugo-themes-by-num-stars.html\">(Switch-to-Order-by-Num-Stars)</a></div>\n"
bycommitTable +="\t<div class='container'>\n\t\t<table class='table monospace'>\n\t\t\t<thead><tr><th scope='col'>Name</th><th scope='col'>Date</th></tr></thead>\n\t\t\t<tbody>\n"
for theme in hugo_themes_list:
name = theme[0].split('/')[1]
row = "\t\t\t\t<tr><td scope='row'>"+"<a href="+theme[4]+">"+name+"</a>""</td>"
row += "<td nowrap>"+theme[2][:10]+"</td>"
row += "<td align='right' nowrap style='padding-left:1em;'>"+str(theme[3])+'&#x2605'+"</td>"
row += "<td align='right' style='padding-left:1em;'>"+theme[1][:6]+"</td></tr>\n"
bycommitTable += row
bycommitTable += "\t\t\t</tbody>\n\t\t</table>\n\t</div>\n</html>"
by_date = open('hugo-themes-report/hugo-themes-by-last-commit-date.html','w')
by_date.write(bycommitTable)
by_date.close()
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
themes_bydate_list = cursor.fetchall()
cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by stargazers_count DESC",(THEMESLISTREPO,))
hugo_themes_list = cursor.fetchall()
themes_bystars_list = cursor.fetchall()
themes = [theme[0] for theme in themes_bystars_list]
tags_list = [('all',str(themes),len(themes))]
cursor.execute("select tag,theme_list,num_themes from tags where num_themes > 2 order by num_themes DESC")
for row in cursor:
tags_list.append(row)
cursor.close()
dbconnection.close()
bystarsTable = "<!DOCTYPE html>\n<html lang='en'>\n"
bystarsTable += "\t<head>\n\t\t<title>Hugo Themes Report Order By Num Stars</title>\n\t\t<meta charset='utf-8'>"
bystarsTable += "\n\t\t<meta name='viewport' content='width=device-width, initial-scale=1'>\n"
bystarsTable += "\t\t<link rel='stylesheet' type='text/css' href='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css' integrity='sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u' crossorigin='anonymous'>\n\t</head>\n"
bystarsTable += "\t<div style='text-align:center;font-size:200%;'><a style='text-decoration:none;'href=\"hugo-themes-by-last-commit-date.html\">(Switch-Order-by-Commit-Date)</a></div>\n"
bystarsTable += "\t<div class='container'>\n\t\t<table class='table monospace'>\n\t\t\t<thead><tr><th scope='col'>Name</th><th scope='col'>Date</th></tr></thead>\n\t\t\t<tbody>\n"
for theme in hugo_themes_list:
name = theme[0].split('/')[1]
row = "\t\t\t\t<tr><td scope='row'>"+"<a href="+theme[4]+">"+name+"</a>""</td>"
row += "<td nowrap>"+theme[2][:10]+"</td>"
row += "<td align='right' nowrap style='padding-left:1em;'>"+str(theme[3])+'&#x2605'+"</td>"
row += "<td align='right' style='padding-left:1em;'>"+theme[1][:6]+"</td></tr>\n"
bystarsTable += row
bystarsTable += "\t\t\t</tbody>\n\t\t</table>\n\t</div>\n</html>"
by_date = open('hugo-themes-report/hugo-themes-by-num-stars.html','w')
by_date.write(bystarsTable)
reportpage = "<!DOCTYPE html>\n<html lang='en'>\n"
reportpage += "\t<head>\n\t\t<title>Hugo Themes Report</title>\n\t\t<meta charset='utf-8'>"
reportpage += "\n\t\t<meta name='viewport' content='width=device-width, initial-scale=1'>\n"
reportpage += "\t\t<link rel='stylesheet' href='https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css' "
reportpage += "integrity='sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm' crossorigin='anonymous'>\n\t</head>\n"
reportpage += "\t<body>\n"
reportpage += make_section(('all',themes_bydate_list,themes_bystars_list,len(themes_bydate_list)))
reportpage += make_buttons(tags_list)
for tag in tags_list[1:]:
tag_theme_list = literal_eval(tag[1])
bydate_list = [theme for theme in themes_bydate_list if theme[0] in tag_theme_list]
bystars_list = [theme for theme in themes_bystars_list if theme[0] in tag_theme_list]
reportpage += make_section((tag[0],bydate_list,bystars_list,tag[2]))
reportpage += "\t</body>"
reportpage += "\n</html>"
by_date = open('hugo-themes-report/hugo-themes-report.html','w')
by_date.write(reportpage)
by_date.close()
@ -227,4 +371,8 @@ if __name__=="__main__":
parse_hugo_themes_list()
get_commit_info_for_hugo_themes()
get_stargazer_count_for_hugo_themes()
get_theme_dot_toml_for_each_hugo_themes()
update_tags_list_for_each_hugo_themes()
write_reports()
"""
"""