diff --git a/rank_hugo_themes.py b/rank_hugo_themes.py
index 7b92469..451af95 100755
--- a/rank_hugo_themes.py
+++ b/rank_hugo_themes.py
@@ -1,16 +1,20 @@
-#!/usr/bin/env python36
+#!/usr/bin/env python3
# rank_hugo_themes.py
-# import sqlite3,json,sys,calendar,time,requests
+import re
+import toml
from calendar import timegm
from time import strptime
from requests import get
from sqlite3 import connect
from json import loads as json_loads
from sys import argv as sys_argv
+from base64 import b64decode
+from ast import literal_eval
DATABASENAME='hugothemes.db'
-# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int);
+# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int);
+# CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int);
THEMESLISTREPO='gohugoio/hugoThemes'
if len(sys_argv) == 2:
@@ -61,10 +65,11 @@ def clean_up():
themes_json = json_loads(themes_json_string)
new_name_list = []
for theme in themes_json:
- if 'gohugoio' not in theme['git_url']:
- split_html_url = theme['html_url'].split('/')
- new_short_name = split_html_url[3]+'/'+split_html_url[4]
- new_name_list.append(new_short_name)
+ if theme['git_url']:
+ if 'gohugoio' not in theme['git_url']:
+ split_html_url = theme['html_url'].split('/')
+ new_short_name = split_html_url[3]+'/'+split_html_url[4]
+ new_name_list.append(new_short_name)
cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,))
old_name_list = cursor.fetchall()
for name in old_name_list:
@@ -82,15 +87,16 @@ def parse_hugo_themes_list():
themes_json_string = cursor.fetchone()[0]
themes_json = json_loads(themes_json_string)
for x in themes_json:
- if 'gohugoio' not in x['git_url']:
- theme_git_url = x['html_url'][:-46]
- theme_git_name = theme_git_url[19:]
- cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name])
- dbconnection.commit()
- sql = "update hugothemes set url=?,commit_sha=? where name=?"
- values=(theme_git_url,x['sha'],theme_git_name)
- cursor.execute(sql,values)
- dbconnection.commit()
+ if x['git_url']:
+ if 'gohugoio' not in x['git_url']:
+ theme_git_url = x['html_url'][:-46]
+ theme_git_name = theme_git_url[19:]
+ cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name])
+ dbconnection.commit()
+ sql = "update hugothemes set url=?,commit_sha=? where name=?"
+ values=(theme_git_url,x['sha'],theme_git_name)
+ cursor.execute(sql,values)
+ dbconnection.commit()
cursor.close()
dbconnection.close()
@@ -171,53 +177,191 @@ def get_stargazer_count_for_hugo_themes():
cursor.close()
dbconnection.close()
+
+def get_theme_dot_toml_for_each_hugo_themes():
+ dbconnection = connect(DATABASENAME)
+ cursor = dbconnection.cursor()
+ cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,))
+ hugo_themes_list = cursor.fetchall()
+ for theme in hugo_themes_list:
+ name,old_ETag = theme
+ api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml"
+
+ if old_ETag != None:
+ headers['If-None-Match'] = old_ETag
+ else:
+ if 'If-None-Match' in headers:
+ del headers['If-None-Match']
+
+ if len(headers) == 0:
+ response = get(api_call_url)
+ else:
+ response = get(api_call_url, headers=headers)
+ if response.status_code == 200:
+ themes_toml_ETag = response.headers['ETag']
+ result = response.json()
+ themes_toml_content = result['content']
+ sql = "update hugothemes set themes_toml_ETag=?,themes_toml_content=? where name=?"
+ values=(themes_toml_ETag,themes_toml_content,name)
+ cursor.execute(sql,values)
+ dbconnection.commit()
+ elif response.status_code == 403:
+ print(response.status_code)
+ cursor.close()
+ dbconnection.close()
+ write_reports()
+ quit()
+ print(response.status_code)
+ cursor.close()
+ dbconnection.close()
+
+
+def update_tags_list_for_each_hugo_themes():
+ dbconnection = connect(DATABASENAME)
+ cursor = dbconnection.cursor()
+ cursor.execute("select name,themes_toml_content from hugothemes where name!=?",(THEMESLISTREPO,))
+ hugo_themes_list = cursor.fetchall()
+ for theme in hugo_themes_list:
+ if theme[1] != None:
+ content = b64decode(theme[1]).decode('utf-8')
+ # put quotes around any unquoted double-dotted version numbers
+ # because python toml libraries will error out on those
+ theme_toml = toml.loads(re.sub(r'\s(\d+\.\d+\.\d+)\s',r'"\1"',content))
+ if 'tags' in theme_toml:
+ if len(theme_toml['tags']) > 0:
+ theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0]
+ num_tags = len(theme_tags)
+ if num_tags > 0:
+ tag_list = str(theme_tags)
+ else:
+ tag_list = None
+ else:
+ tag_list = None
+ num_tags = 0
+ else:
+ tag_list = None
+ num_tags = 0
+ else:
+ tag_list = None
+ num_tags = 0
+ sql = "update hugothemes set tags_list=?,num_tags=? where name=?"
+ values=(tag_list,num_tags,theme[0])
+ cursor.execute(sql,values)
+ dbconnection.commit()
+ cursor.close()
+ dbconnection.close()
+
+
+def update_tag_table():
+ dbconnection = connect(DATABASENAME)
+ cursor = dbconnection.cursor()
+ cursor.execute("select name,tags_list from hugothemes where name!=?",(THEMESLISTREPO,))
+ hugo_themes_list = cursor.fetchall()
+ tags_list = []
+ for theme in hugo_themes_list:
+ if theme[1] != None:
+ tags = literal_eval(theme[1])
+ for tag in tags:
+ if len(tag) > 0:
+ if tag not in tags_list:
+ tags_list.append(tag)
+
+ for tag in tags_list:
+ cursor.execute("insert or ignore into tags (tag) values (?)",[tag])
+ dbconnection.commit()
+ theme_list = []
+ for theme in hugo_themes_list:
+ if theme[1] != None:
+ tags = literal_eval(theme[1])
+ if tag in tags:
+ theme_list.append(theme[0])
+ sql = "update tags set theme_list=?,num_themes=? where tag=?"
+ values=(str(theme_list),len(theme_list),tag)
+ cursor.execute(sql,values)
+ dbconnection.commit()
+
+ cursor.close()
+ dbconnection.close()
+
+
+def make_buttons(tags_list):
+ button_block = "\t\t
\n"
+ for tag in tags_list:
+ button_block += "\t\t\t
\n"
+ button_block += "\t\t
\n"
+ return button_block
+
+
+def make_nav_buttons(button_info):
+ button_block = "\t\t\n"
+ button_block += "\t\t\t
\n"
+ if button_info[2] > 10:
+ button_block += "\t\t\t
\n"
+ button_block += "\t\t\t
\n"
+ button_block += "\t\t
\n"
+ return button_block
+
+
+
+def make_table(themes_info):
+ table = "\t\n\t\t
\n\t\t\t" + themes_info[1] + " (tag) " + themes_info[2] + " | Date |
\n\t\t\t\n"
+ for theme in themes_info[0]:
+ name = theme[0].split('/')[1]
+ row = "\t\t\t\t"+""+name+""" | "
+ row += ""+theme[2][:10]+" | "
+ row += ""+str(theme[3])+'★'+" | "
+ row += ""+theme[1][:6]+" |
\n"
+ table += row
+ table += "\t\t\t\n\t\t
\n\t
\n"
+ return table
+
+
+def make_section(section_info):
+ if section_info[3] > 10:
+ section = make_nav_buttons((section_info[0],'by-date',section_info[3]))
+ section += make_table((section_info[1],section_info[0],'by-date'))
+ section += make_nav_buttons((section_info[0],'by-stars',section_info[3]))
+ section += make_table((section_info[2],section_info[0],'by-stars'))
+ else:
+ section = make_nav_buttons((section_info[0],'by-date',section_info[3]))
+ section += make_table((section_info[1],section_info[0],'by-date'))
+ return section
+
+
def write_reports():
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by commit_date_in_seconds DESC",(THEMESLISTREPO,))
- hugo_themes_list = cursor.fetchall()
- cursor.close()
- dbconnection.close()
- bycommitTable = "\n\n"
- bycommitTable += "\t\n\t\tHugo Themes Report Order By Commit Date\n\t\t"
- bycommitTable += "\n\t\t\n"
- bycommitTable += "\t\t\n\t\n"
- bycommitTable += "\t\n"
- bycommitTable +="\t\n\t\t
\n\t\t\tName | Date |
\n\t\t\t\n"
- for theme in hugo_themes_list:
- name = theme[0].split('/')[1]
- row = "\t\t\t\t"+""+name+""" | "
- row += ""+theme[2][:10]+" | "
- row += ""+str(theme[3])+'★'+" | "
- row += ""+theme[1][:6]+" |
\n"
- bycommitTable += row
- bycommitTable += "\t\t\t\n\t\t
\n\t
\n"
- by_date = open('hugo-themes-report/hugo-themes-by-last-commit-date.html','w')
- by_date.write(bycommitTable)
- by_date.close()
-
- dbconnection = connect(DATABASENAME)
- cursor = dbconnection.cursor()
+ themes_bydate_list = cursor.fetchall()
cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by stargazers_count DESC",(THEMESLISTREPO,))
- hugo_themes_list = cursor.fetchall()
+ themes_bystars_list = cursor.fetchall()
+ themes = [theme[0] for theme in themes_bystars_list]
+ tags_list = [('all',str(themes),len(themes))]
+ cursor.execute("select tag,theme_list,num_themes from tags where num_themes > 2 order by num_themes DESC")
+ for row in cursor:
+ tags_list.append(row)
cursor.close()
dbconnection.close()
- bystarsTable = "\n\n"
- bystarsTable += "\t\n\t\tHugo Themes Report Order By Num Stars\n\t\t"
- bystarsTable += "\n\t\t\n"
- bystarsTable += "\t\t\n\t\n"
- bystarsTable += "\t\n"
- bystarsTable += "\t\n\t\t
\n\t\t\tName | Date |
\n\t\t\t\n"
- for theme in hugo_themes_list:
- name = theme[0].split('/')[1]
- row = "\t\t\t\t"+""+name+""" | "
- row += ""+theme[2][:10]+" | "
- row += ""+str(theme[3])+'★'+" | "
- row += ""+theme[1][:6]+" |
\n"
- bystarsTable += row
- bystarsTable += "\t\t\t\n\t\t
\n\t
\n"
- by_date = open('hugo-themes-report/hugo-themes-by-num-stars.html','w')
- by_date.write(bystarsTable)
+
+ reportpage = "\n\n"
+ reportpage += "\t\n\t\tHugo Themes Report\n\t\t"
+ reportpage += "\n\t\t\n"
+ reportpage += "\t\t\n\t\n"
+ reportpage += "\t\n"
+
+ reportpage += make_section(('all',themes_bydate_list,themes_bystars_list,len(themes_bydate_list)))
+ reportpage += make_buttons(tags_list)
+ for tag in tags_list[1:]:
+ tag_theme_list = literal_eval(tag[1])
+ bydate_list = [theme for theme in themes_bydate_list if theme[0] in tag_theme_list]
+ bystars_list = [theme for theme in themes_bystars_list if theme[0] in tag_theme_list]
+ reportpage += make_section((tag[0],bydate_list,bystars_list,tag[2]))
+
+ reportpage += "\t"
+ reportpage += "\n"
+ by_date = open('hugo-themes-report/hugo-themes-report.html','w')
+ by_date.write(reportpage)
by_date.close()
@@ -227,4 +371,8 @@ if __name__=="__main__":
parse_hugo_themes_list()
get_commit_info_for_hugo_themes()
get_stargazer_count_for_hugo_themes()
+ get_theme_dot_toml_for_each_hugo_themes()
+ update_tags_list_for_each_hugo_themes()
write_reports()
+ """
+ """