rank_hugo_themes.py implement sqlalchemy instead of sqlite3

This commit is contained in:
Trent Palmer 2019-01-22 02:30:14 -08:00
parent 69577a605f
commit 25d2cb64bd
2 changed files with 275 additions and 320 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
hugothemes.db
__pycache__
hugo-themes-report

View File

@ -6,89 +6,140 @@ import toml
from calendar import timegm from calendar import timegm
from time import strptime from time import strptime
from requests import get from requests import get
from sqlite3 import connect
from json import loads as json_loads from json import loads as json_loads
from sys import argv as sys_argv from sys import argv as sys_argv
from base64 import b64decode from base64 import b64decode
from ast import literal_eval from ast import literal_eval
DATABASENAME='hugothemes.db' from sqlalchemy import create_engine,Column,Integer,VARCHAR,TEXT
# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int, dot_gitmodules_content text); from sqlalchemy.ext.declarative import declarative_base
# CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int); from sqlalchemy.orm import deferred,sessionmaker
# CREATE TABLE hugothemes_from_gitlab (name varchar unique, url text, commit_sha text, id text, commit_date_in_seconds int, commit_date text, star_count int, themes_toml_content text);
THEMESLISTREPO='gohugoio/hugoThemes' engine = create_engine('sqlite:///hugothemes.db',echo=False)
Base = declarative_base()
class Tags(Base):
__tablename__ = 'tags'
tag = Column(VARCHAR,primary_key=True)
theme_list = Column(TEXT)
num_themes = Column(Integer)
def __repr__(self):
repr_string = "<(tag = '%s', theme_list = '%s', num_themes = '%s')>"
repr_values = (self.tag,self.theme_list,self.num_themes)
return repr_string % repr_values
class Hugothemes_from_gitlab(Base):
__tablename__ = 'hugothemes_from_gitlab'
name = Column(VARCHAR,primary_key=True)
url = Column(TEXT)
commit_sha = Column(TEXT)
gitlab_id = Column(TEXT)
commit_date_in_seconds = Column(Integer)
commit_date = Column(TEXT)
star_count = Column(Integer)
themes_toml_content = Column(TEXT)
def __repr__(self):
repr_string = "<(name = '%s', url = '%s', commit_sha = '%s', gitlab_id = '%s', commit_date_in_seconds = '%s'"
repr_string += ", commit_date = '%s', star_count = '%s', themes_toml_content = '%s')>"
repr_values = (self.name,self.commit_sha,self.gitlab_id,self.commit_date_in_seconds,self.commit_date,
self.star_count,self.themes_toml_content)
return repr_string % repr_values
class Hugothemes(Base):
__tablename__ = 'hugothemes'
name = Column(VARCHAR,primary_key=True)
ETag = Column(TEXT)
url = Column(TEXT)
jsondump = deferred(Column(TEXT))
commit_sha = Column(TEXT)
commit_date = Column(TEXT)
commit_date_in_seconds = Column(Integer)
repo_ETag = Column(TEXT)
stargazers_count = Column(Integer)
themes_toml_ETag = Column(TEXT)
themes_toml_content = deferred(Column(TEXT))
tags_list = Column(TEXT)
num_tags = Column(Integer)
dot_gitmodules_content = Column(TEXT)
def __repr__(self):
repr_string = "<(name = '%s', ETag = '%s', url = '%s', jsondump = '%s', commit_sha = '%s', commit_date = '%s'"
repr_string += ", commit_date_in_seconds = '%s', repo_ETag = '%s', stargazers_count = '%s', themes_toml_ETag = '%s'"
repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', dot_gitmodule_content = '%s')>"
repr_values = (self.name,self.ETag,self.url,self.jsondump,self.commit_sha,self.commit_date,self.commit_date_in_seconds,
self.repo_ETag,self.stargazers_count,self.themes_toml_ETag,self.themes_toml_content,self.tags_list,
self.num_tags,self.dot_gitmodules_content)
return repr_string % repr_values
THEMESLISTREPO = 'gohugoio/hugoThemes'
if len(sys_argv) == 2: if len(sys_argv) == 2:
headers = { 'Authorization' : 'token '+sys_argv[1] } headers = { 'Authorization' : 'token ' + sys_argv[1] }
else: else:
headers = {} headers = {}
def get_hugo_themes_list(): def get_hugo_themes_list():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes_list_repo = session.query(Hugothemes).filter_by(name=THEMESLISTREPO).first()
cursor.execute("select ETag from hugothemes where name=?",(THEMESLISTREPO,))
old_ETag = cursor.fetchone()[0]
api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents" api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents"
if themes_list_repo.url != api_call_url: themes_list_repo.url = api_call_url
if old_ETag != None: if themes_list_repo.ETag != None:
headers['If-None-Match'] = old_ETag headers['If-None-Match'] = themes_list_repo.ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers: del headers['If-None-Match']
del headers['If-None-Match']
if len(headers) == 0: if len(headers) == 0:
response = get(api_call_url) response = get(api_call_url)
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
ETag = response.headers['ETag'] themes_list_repo.ETag = response.headers['ETag'].lstrip('W/')
sql = "update hugothemes set ETag=?,url=?,jsondump=? where name=?" themes_list_repo.jsondump = response.text
values=(ETag,api_call_url,response.text,THEMESLISTREPO) session.commit()
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_hugo_themes_list.__name__)
cursor.close() session.commit()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_hugo_themes_list.__name__)
cursor.close()
dbconnection.close()
def update_hugo_themes_submodule_list(): def update_hugo_themes_submodule_list():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes_list_repo = session.query(Hugothemes).filter_by(name=THEMESLISTREPO).first()
cursor.execute("select jsondump,dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,)) themes_json = json_loads(themes_list_repo.jsondump)
themes_json_string,dot_gitmodule_json_string = cursor.fetchone() if themes_list_repo.dot_gitmodules_content:
themes_json = json_loads(themes_json_string) dot_gitmodule_json = json_loads(themes_list_repo.dot_gitmodules_content)
if dot_gitmodule_json_string:
dot_gitmodule_json = json_loads(dot_gitmodule_json_string)
for theme in themes_json: for theme in themes_json:
if theme['name'] == '.gitmodules': if theme['name'] == '.gitmodules':
new_gitmodules_sha = theme['sha'] new_gitmodules_sha = theme['sha']
break break
if not dot_gitmodule_json_string or new_gitmodules_sha != dot_gitmodule_json['sha']: if (not dot_gitmodule_json) or (new_gitmodules_sha != dot_gitmodule_json['sha']):
api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents/.gitmodules" api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents/.gitmodules"
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
sql = "update hugothemes set dot_gitmodules_content=? where name=?" if themes_list_repo.dot_gitmodules_content != response.text:
values=(response.text,THEMESLISTREPO) themes_list_repo.dot_gitmodules_content = response.text
cursor.execute(sql,values) session.commit()
dbconnection.commit() print(response.status_code,update_hugo_themes_submodule_list.__name__)
print(response.status_code)
cursor.close()
dbconnection.close()
def get_hugo_themes_submodule_list(): def get_hugo_themes_submodule_list():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() dot_gitmodule_json_string = session.query(Hugothemes.dot_gitmodules_content).filter_by(name=THEMESLISTREPO).first()
cursor.execute("select dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,)) dot_gitmodule_json = json_loads(dot_gitmodule_json_string[0])
dot_gitmodule_json_string = cursor.fetchone()[0]
dot_gitmodule_json = json_loads(dot_gitmodule_json_string)
dot_gitmodule_content = b64decode(dot_gitmodule_json['content']).decode('utf-8').replace('\n\n','\n').split('\n') dot_gitmodule_content = b64decode(dot_gitmodule_json['content']).decode('utf-8').replace('\n\n','\n').split('\n')
submodules = [] submodules = []
for line in range(len(dot_gitmodule_content[:-1])): for line in range(len(dot_gitmodule_content[:-1])):
@ -99,55 +150,38 @@ def get_hugo_themes_submodule_list():
if 'url = ' in dot_gitmodule_content[line+2]: if 'url = ' in dot_gitmodule_content[line+2]:
url = re.match(r'^.*(url = )(.*)$',dot_gitmodule_content[line+2]).group(2) url = re.match(r'^.*(url = )(.*)$',dot_gitmodule_content[line+2]).group(2)
submodules.append((name,path,url)) submodules.append((name,path,url))
cursor.close()
dbconnection.close()
return submodules return submodules
def clean_up(): def clean_up():
dbconnection = connect(DATABASENAME) new_name_list = set()
cursor = dbconnection.cursor()
new_name_list = []
submodules = get_hugo_themes_submodule_list() submodules = get_hugo_themes_submodule_list()
for submodule in submodules: for submodule in submodules:
submodule_name = re.sub(r'(^https://github.com/|^https://gitlab.com/|\.git$)','',submodule[2]) submodule_name = re.sub(r'(^https://github.com/|^https://gitlab.com/|\.git$)','',submodule[2])
new_name_list.append(submodule_name) new_name_list.add(submodule_name)
cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,)) session = sessionmaker(bind=engine)()
old_name_list = [name[0] for name in cursor] old_name_list = [theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all()]
for name in old_name_list: for name in old_name_list:
if name not in new_name_list: if name not in new_name_list:
cursor.execute("delete from hugothemes where name=?",(name,)) removed_theme = session.query(Hugothemes).filter_by(name=name).first()
dbconnection.commit() session.delete(removed_theme)
cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) themes_json_string = session.query(Hugothemes.jsondump).filter_by(name=THEMESLISTREPO).first()
themes_json_string = cursor.fetchone()[0] themes_json = json_loads(themes_json_string[0])
themes_json = json_loads(themes_json_string) new_name_list = set()
new_name_list = []
for theme in themes_json: for theme in themes_json:
if theme['git_url']: if theme['git_url']:
if 'gohugoio' not in theme['git_url']: if 'gohugoio' not in theme['git_url']:
split_html_url = theme['html_url'].split('/') split_html_url = theme['html_url'].split('/')
new_short_name = split_html_url[3]+'/'+split_html_url[4] new_short_name = split_html_url[3]+'/'+split_html_url[4]
new_name_list.append(new_short_name) new_name_list.add(new_short_name)
for name in old_name_list: for name in old_name_list:
if name not in new_name_list: if name not in new_name_list:
cursor.execute("delete from hugothemes where name=?",(name,)) removed_theme = session.query(Hugothemes).filter_by(name=name).first()
dbconnection.commit() if removed_theme != None: session.delete(removed_theme)
session.commit()
cursor.close()
dbconnection.close()
def get_non_github_themes():
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
cursor.execute("select name from hugothemes_from_gitlab;")
non_github_theme_list = [theme[0] for theme in cursor]
cursor.close()
dbconnection.close()
return non_github_theme_list
def parse_submodules_from_gitlab(): def parse_submodules_from_gitlab():
@ -157,13 +191,9 @@ def parse_submodules_from_gitlab():
if 'gitlab' in submodule[2]: if 'gitlab' in submodule[2]:
temp_submodules_list.append(submodule) temp_submodules_list.append(submodule)
if (len(temp_submodules_list) > 0): if (len(temp_submodules_list) > 0):
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes_json_string = session.query(Hugothemes.jsondump).filter_by(name=THEMESLISTREPO).first()
cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) themes_json = json_loads(themes_json_string[0])
themes_json_string = cursor.fetchone()[0]
themes_json = json_loads(themes_json_string)
cursor.close()
dbconnection.close()
submodules_list = [] submodules_list = []
for submodule in temp_submodules_list: for submodule in temp_submodules_list:
for theme in themes_json: for theme in themes_json:
@ -171,75 +201,61 @@ def parse_submodules_from_gitlab():
if (theme['html_url'] == None) or (theme['git_url'] == None): if (theme['html_url'] == None) or (theme['git_url'] == None):
submodules_list.append((submodule[0],submodule[1],submodule[2],theme['sha'])) submodules_list.append((submodule[0],submodule[1],submodule[2],theme['sha']))
if (len(submodules_list) > 0): if (len(submodules_list) > 0):
dbconnection = connect(DATABASENAME)
cursor = dbconnection.cursor()
for submodule in submodules_list: for submodule in submodules_list:
url = re.sub(r'\.git$','',submodule[2]) url = re.sub(r'\.git$','',submodule[2])
theme_name = url[19:] theme_name = url[19:]
cursor.execute("insert or ignore into hugothemes_from_gitlab (name) values (?)",[theme_name]) theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme_name).first()
dbconnection.commit() if theme == None:
sql = "update hugothemes_from_gitlab set url=?,commit_sha=? where name=?" session.add(Hugothemes_from_gitlab(name=theme_name,url=url,commit_sha=submodule[3]))
values=(url,submodule[3],theme_name) else:
cursor.execute(sql,values) if theme.url != url: theme.url = url
dbconnection.commit() if theme.commit_sha != submodule[3]: theme.commit_sha = submodule[3]
cursor.close() session.commit()
dbconnection.close()
def parse_hugo_themes_list(): def parse_hugo_themes_list():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes_json_string = session.query(Hugothemes.jsondump).filter_by(name=THEMESLISTREPO).first()
cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,)) themes_json = json_loads(themes_json_string[0])
themes_json_string = cursor.fetchone()[0]
themes_json = json_loads(themes_json_string)
for x in themes_json: for x in themes_json:
if x['git_url']: if x['git_url']:
if 'gohugoio' not in x['git_url']: if 'gohugoio' not in x['git_url']:
theme_git_url = x['html_url'][:-46] theme_git_url = x['html_url'][:-46]
theme_git_name = theme_git_url[19:] theme_git_name = theme_git_url[19:]
cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme_git_name]) theme = session.query(Hugothemes).filter_by(name=theme_git_name).first()
dbconnection.commit() if theme == None:
sql = "update hugothemes set url=?,commit_sha=? where name=?" session.add(Hugothemes(name=theme_git_name,url=theme_git_url,commit_sha=x['sha']))
values=(theme_git_url,x['sha'],theme_git_name) else:
cursor.execute(sql,values) if theme.url != theme_git_url: theme.url = theme_git_url
dbconnection.commit() if theme.commit_sha != x['sha']: theme.commit_sha = x['sha']
cursor.close() session.commit()
dbconnection.close()
def get_gitlab_project_ids(): def get_gitlab_project_ids():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = (theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all())
cursor.execute("select name,url,id from hugothemes_from_gitlab;") match = re.compile(r'(Project ID: )(\d{5,})')
themes = [row for row in cursor]
for theme in themes: for theme in themes:
if theme[2] == None: gitlab_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).one()
response = get(theme[1]) if gitlab_theme.gitlab_id == None:
id_result = re.search(r'(Project ID: )(\d{5,})',response.text).group(2) response = get(gitlab_theme.url)
sql = "update hugothemes_from_gitlab set id=? where name=?" gitlab_theme.gitlab_id = match.search(response.text).group(2)
values=(id_result,theme[0]) session.commit()
cursor.execute(sql,values)
dbconnection.commit()
cursor.close()
dbconnection.close()
return True return True
def get_commit_info_for_hugo_themes(): def get_commit_info_for_hugo_themes():
non_github_theme_list = get_non_github_themes() session = sessionmaker(bind=engine)()
themes = [] non_github_theme_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
dbconnection = connect(DATABASENAME) themes = set(theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all())
cursor = dbconnection.cursor() for theme in non_github_theme_list:
cursor.execute("select name,ETag,url,commit_sha from hugothemes where name!=?",(THEMESLISTREPO,)) themes -= {theme}
for theme in cursor: for hugo_theme in themes:
if theme[0] not in non_github_theme_list: theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
themes.append(theme) api_call_url = 'https://api.github.com/repos/' + theme.name + '/commits/' + theme.commit_sha
for theme in themes:
name,old_ETag,url,commit_sha = theme
api_call_url = 'https://api.github.com/repos/' + name + '/commits/' + commit_sha
if old_ETag != None: if theme.ETag != None:
headers['If-None-Match'] = old_ETag headers['If-None-Match'] = theme.ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers:
del headers['If-None-Match'] del headers['If-None-Match']
@ -249,70 +265,52 @@ def get_commit_info_for_hugo_themes():
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
ETag = response.headers['ETag'] theme.ETag = response.headers['ETag'].lstrip('W/')
result = response.json() result = response.json()
commit_date = result['commit']['author']['date'] theme.commit_date = result['commit']['author']['date']
commit_date_in_seconds = timegm(strptime(commit_date,'%Y-%m-%dT%H:%M:%SZ')) theme.commit_date_in_seconds = timegm(strptime(theme.commit_date,'%Y-%m-%dT%H:%M:%SZ'))
sql = "update hugothemes set ETag=?,commit_date=?,commit_date_in_seconds=? where name=?" session.commit()
values=(ETag,commit_date,commit_date_in_seconds,name)
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_commit_info_for_hugo_themes.__name__)
cursor.close()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_commit_info_for_hugo_themes.__name__)
cursor.close()
dbconnection.close()
def get_commit_info_for_hugo_themes_from_gitlab(): def get_commit_info_for_hugo_themes_from_gitlab():
if get_gitlab_project_ids(): if get_gitlab_project_ids():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
cursor.execute("select name,url,commit_sha,id from hugothemes_from_gitlab") match = re.compile(r'(\.\d{3})(Z$)')
hugo_themes_list = cursor.fetchall() for hugo_theme in themes:
for theme in hugo_themes_list: theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
name,url,commit_sha,gitlab_project_id = theme api_call_url = 'https://gitlab.com/api/v4/projects/' + theme.gitlab_id + '/repository/commits/' + theme.commit_sha
api_call_url = 'https://gitlab.com/api/v4/projects/' + gitlab_project_id + '/repository/commits/' + commit_sha
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
commit_date = re.sub(r'(\.\d{3})(Z$)',r'\2',result['created_at']) theme.commit_date = match.sub(r'\2',result['created_at'])
commit_date_in_seconds = timegm(strptime(commit_date,'%Y-%m-%dT%H:%M:%SZ')) theme.commit_date_in_seconds = timegm(strptime(theme.commit_date,'%Y-%m-%dT%H:%M:%SZ'))
sql = "update hugothemes_from_gitlab set commit_date=?,commit_date_in_seconds=? where name=?" session.commit()
values=(commit_date,commit_date_in_seconds,name)
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_commit_info_for_hugo_themes_from_gitlab.__name__)
cursor.close()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_commit_info_for_hugo_themes_from_gitlab.__name__)
cursor.close()
dbconnection.close()
def get_stargazer_count_for_hugo_themes(): def get_stargazer_count_for_hugo_themes():
non_github_theme_list = get_non_github_themes() session = sessionmaker(bind=engine)()
themes = [] non_github_theme_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
dbconnection = connect(DATABASENAME) themes = set(theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all())
cursor = dbconnection.cursor() for theme in non_github_theme_list:
cursor.execute("select name,repo_ETag from hugothemes where name!=?",(THEMESLISTREPO,)) themes -= {theme}
for theme in cursor: for hugo_theme in themes:
if theme[0] not in non_github_theme_list: theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
themes.append(theme) api_call_url = 'https://api.github.com/repos/' + theme.name
for theme in themes:
name,old_ETag = theme
api_call_url = 'https://api.github.com/repos/' + name
if old_ETag != None: if theme.repo_ETag != None:
headers['If-None-Match'] = old_ETag headers['If-None-Match'] = theme.repo_ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers:
del headers['If-None-Match'] del headers['If-None-Match']
@ -322,67 +320,48 @@ def get_stargazer_count_for_hugo_themes():
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
repo_ETag = response.headers['ETag'] theme.repo_ETag = response.headers['ETag'].lstrip('W/')
result = response.json() result = response.json()
stargazers_count = result['stargazers_count'] theme.stargazers_count = result['stargazers_count']
sql = "update hugothemes set repo_ETag=?,stargazers_count=? where name=?" session.commit()
values=(repo_ETag,stargazers_count,name)
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_stargazer_count_for_hugo_themes.__name__)
cursor.close()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_stargazer_count_for_hugo_themes.__name__)
cursor.close()
dbconnection.close()
def get_stargazer_count_for_hugo_themes_from_gitlab(): def get_stargazer_count_for_hugo_themes_from_gitlab():
if get_gitlab_project_ids(): if get_gitlab_project_ids():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = [(theme,gitlab_id) for theme,gitlab_id in session.query(Hugothemes_from_gitlab.name,Hugothemes_from_gitlab.gitlab_id).all()]
cursor.execute("select name,id from hugothemes_from_gitlab;") for hugo_theme in themes:
themes = cursor.fetchall() theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme[0]).one()
for theme in themes: api_call_url = 'https://gitlab.com/api/v4/projects/' + hugo_theme[1]
api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1]
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
star_count = result['star_count'] if theme.star_count != result['star_count']:
sql = "update hugothemes_from_gitlab set star_count=? where name=?" theme.star_count = result['star_count']
values=(star_count,theme[0]) session.commit()
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_stargazer_count_for_hugo_themes_from_gitlab.__name__)
cursor.close()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_stargazer_count_for_hugo_themes_from_gitlab.__name__)
cursor.close()
dbconnection.close()
def get_theme_dot_toml_for_each_hugo_themes(): def get_theme_dot_toml_for_each_hugo_themes():
non_github_theme_list = get_non_github_themes() session = sessionmaker(bind=engine)()
themes = [] non_github_theme_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
dbconnection = connect(DATABASENAME) themes = set(theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all())
cursor = dbconnection.cursor() for theme in non_github_theme_list: themes -= {theme}
cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,)) for hugo_theme in themes:
for theme in cursor: theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
if theme[0] not in non_github_theme_list: api_call_url = "https://api.github.com/repos/"+theme.name+"/contents/theme.toml"
themes.append(theme)
for theme in themes:
name,old_ETag = theme
api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml"
if old_ETag != None: if theme.themes_toml_ETag != None:
headers['If-None-Match'] = old_ETag headers['If-None-Match'] = theme.themes_toml_ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers:
del headers['If-None-Match'] del headers['If-None-Match']
@ -392,135 +371,116 @@ def get_theme_dot_toml_for_each_hugo_themes():
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
themes_toml_ETag = response.headers['ETag'] theme.themes_toml_ETag = response.headers['ETag'].lstrip('W/')
result = response.json() result = response.json()
themes_toml_content = result['content'] theme.themes_toml_content = result['content']
sql = "update hugothemes set themes_toml_ETag=?,themes_toml_content=? where name=?" session.commit()
values=(themes_toml_ETag,themes_toml_content,name)
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_theme_dot_toml_for_each_hugo_themes.__name__)
cursor.close()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_theme_dot_toml_for_each_hugo_themes.__name__)
cursor.close()
dbconnection.close()
def get_theme_dot_toml_for_each_hugo_themes_from_gitlab(): def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
if get_gitlab_project_ids(): if get_gitlab_project_ids():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = [(theme,gitlab_id) for theme,gitlab_id in session.query(Hugothemes_from_gitlab.name,Hugothemes_from_gitlab.gitlab_id).all()]
cursor.execute("select name,id from hugothemes_from_gitlab") for hugo_theme in themes:
themes = cursor.fetchall() theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme[0]).one()
for theme in themes: api_call_url = 'https://gitlab.com/api/v4/projects/' + hugo_theme[1] + '/repository/files/theme.toml?ref=master'
api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1] + '/repository/files/theme.toml?ref=master'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
themes_toml_content = result['content'] theme.themes_toml_content = result['content']
sql = "update hugothemes_from_gitlab set themes_toml_content=? where name=?" session.commit()
values=(themes_toml_content,theme[0])
cursor.execute(sql,values)
dbconnection.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code) print(response.status_code,get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
cursor.close()
dbconnection.close()
write_reports() write_reports()
quit() quit()
print(response.status_code) print(response.status_code,get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
cursor.close()
dbconnection.close()
def coalesce_themes(): def coalesce_themes():
if get_gitlab_project_ids(): if get_gitlab_project_ids():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
cursor.execute("select name,url,commit_sha,commit_date,commit_date_in_seconds,star_count,themes_toml_content from hugothemes_from_gitlab") for hugo_theme in themes:
themes = cursor.fetchall() htfgitlab = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
for theme in themes: theme = session.query(Hugothemes).filter_by(name=hugo_theme).first()
cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme[0]]) if theme == None:
dbconnection.commit() session.add(Hugothemes(name=hugo_theme,url=htfgitlab.url,commit_sha=htfgitlab.commit_sha,
sql = "update hugothemes set url=?,commit_sha=?,commit_date=?,commit_date_in_seconds=?,stargazers_count=?,themes_toml_content=? where name=?" commit_date=htfgitlab.commit_date,commit_date_in_seconds=htfgitlab.commit_date_in_seconds,
values=(theme[1],theme[2],theme[3],theme[4],theme[5],theme[6],theme[0]) stargazers_count=htfgitlab.star_count,themes_toml_content=htfgitlab.themes_toml_content))
cursor.execute(sql,values) else:
dbconnection.commit() if theme.url != htfgitlab.url: theme.url = htfgitlab.url
cursor.close() if theme.commit_sha != htfgitlab.commit_sha: theme.commit_sha = htfgitlab.commit_sha
dbconnection.close() if theme.commit_date != htfgitlab.commit_date: theme.commit_date = htfgitlab.commit_date
if theme.commit_date_in_seconds != htfgitlab.commit_date_in_seconds:
theme.commit_date_in_seconds = htfgitlab.commit_date_in_seconds
if theme.stargazers_count != htfgitlab.star_count: theme.stargazers_count = htfgitlab.star_count
theme.themes_toml_content = htfgitlab.themes_toml_content
session.commit()
def update_tags_list_for_each_hugo_themes(): def update_tags_list_for_each_hugo_themes():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = [theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all()]
cursor.execute("select name,themes_toml_content from hugothemes where name!=?",(THEMESLISTREPO,)) match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
hugo_themes_list = cursor.fetchall() for hugo_theme in themes:
for theme in hugo_themes_list: theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
if theme[1] != None: if theme.themes_toml_content != None:
content = b64decode(theme[1]).decode('utf-8') content = b64decode(theme.themes_toml_content).decode('utf-8')
# put quotes around any unquoted double-dotted version numbers # put quotes around any unquoted double-dotted version numbers
# because python toml libraries will error out on those # because python toml libraries will error out on those
theme_toml = toml.loads(re.sub(r'\s(\d+\.\d+\.\d+)\s',r'"\1"',content)) theme_toml = toml.loads(match.sub(r'"\1"',content))
if 'tags' in theme_toml: if 'tags' in theme_toml:
if len(theme_toml['tags']) > 0: if len(theme_toml['tags']) > 0:
theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0] theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0]
num_tags = len(theme_tags) if theme.num_tags != len(theme_tags): theme.num_tags = len(theme_tags)
if num_tags > 0: if theme.num_tags > 0:
tag_list = str(theme_tags) if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags)
else: else:
tag_list = None if theme.tags_list != None: theme.tags_list = None
else: else:
tag_list = None if theme.tags_list != None: theme.tags_list = None
num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
else: else:
tag_list = None if theme.tags_list != None: theme.tags_list = None
num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
else: else:
tag_list = None if theme.tags_list != None: theme.tags_list = None
num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
sql = "update hugothemes set tags_list=?,num_tags=? where name=?" session.commit()
values=(tag_list,num_tags,theme[0])
cursor.execute(sql,values)
dbconnection.commit()
cursor.close()
dbconnection.close()
def update_tag_table(): def update_tag_table():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() themes = [(theme,tags_list) for theme,tags_list in session.query(Hugothemes.name,Hugothemes.tags_list)
cursor.execute("select name,tags_list from hugothemes where name!=?",(THEMESLISTREPO,)) .filter(Hugothemes.name!=THEMESLISTREPO).all()]
hugo_themes_list = cursor.fetchall() tags_list = set()
tags_list = [] for theme in themes:
for theme in hugo_themes_list:
if theme[1] != None: if theme[1] != None:
tags = literal_eval(theme[1]) tags = literal_eval(theme[1])
for tag in tags: for tag in tags:
if len(tag) > 0: if len(tag) > 0:
if tag not in tags_list: tags_list.add(tag)
tags_list.append(tag)
for tag in tags_list: for hugo_tag in tags_list:
cursor.execute("insert or ignore into tags (tag) values (?)",[tag])
dbconnection.commit()
theme_list = [] theme_list = []
for theme in hugo_themes_list: for theme in themes:
if theme[1] != None: if theme[1] != None:
tags = literal_eval(theme[1]) tags = literal_eval(theme[1])
if tag in tags: if hugo_tag in tags:
theme_list.append(theme[0]) theme_list.append(theme[0])
sql = "update tags set theme_list=?,num_themes=? where tag=?" tag = session.query(Tags).filter_by(tag=hugo_tag).first()
values=(str(theme_list),len(theme_list),tag) if tag == None:
cursor.execute(sql,values) session.add(Tag(tag=hugo_tag,theme_list=str(theme_list),num_themes=len(theme_list)))
dbconnection.commit() else:
theme_list,num_themes = str(theme_list),len(theme_list)
cursor.close() if tag.theme_list != theme_list: tag.theme_list = theme_list
dbconnection.close() if tag.num_themes != num_themes: tag.num_themes = num_themes
session.commit()
def make_buttons(tags_list): def make_buttons(tags_list):
@ -567,19 +527,13 @@ def make_section(section_info):
def write_reports(): def write_reports():
dbconnection = connect(DATABASENAME) session = sessionmaker(bind=engine)()
cursor = dbconnection.cursor() u,v,w,x,y,z = Hugothemes.name,Hugothemes.commit_sha,Hugothemes.commit_date,Hugothemes.stargazers_count,Hugothemes.url,Hugothemes.commit_date_in_seconds
cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by commit_date_in_seconds DESC",(THEMESLISTREPO,)) themes_bydate_list = [(vals) for (vals) in session.query(u,v,w,x,y).filter(Hugothemes.name!=THEMESLISTREPO).order_by(z.desc())]
themes_bydate_list = cursor.fetchall() themes_bystars_list = [(vals) for (vals) in session.query(u,v,w,x,y).filter(Hugothemes.name!=THEMESLISTREPO).order_by(x.desc())]
cursor.execute("select name,commit_sha,commit_date,stargazers_count,url from hugothemes where name!=? order by stargazers_count DESC",(THEMESLISTREPO,))
themes_bystars_list = cursor.fetchall()
themes = [theme[0] for theme in themes_bystars_list] themes = [theme[0] for theme in themes_bystars_list]
tags_list = [('all',str(themes),len(themes))] tags_list = [('all',str(themes),len(themes))]
cursor.execute("select tag,theme_list,num_themes from tags where num_themes > 2 order by num_themes DESC") tags_list += [(vals) for (vals) in session.query(Tags.tag,Tags.theme_list,Tags.num_themes).filter(Tags.num_themes > 2).order_by(Tags.num_themes.desc())]
for row in cursor:
tags_list.append(row)
cursor.close()
dbconnection.close()
reportpage = "<!DOCTYPE html>\n<html lang='en'>\n" reportpage = "<!DOCTYPE html>\n<html lang='en'>\n"
reportpage += "\t<head>\n\t\t<title>Hugo Themes Report</title>\n\t\t<meta charset='utf-8'>" reportpage += "\t<head>\n\t\t<title>Hugo Themes Report</title>\n\t\t<meta charset='utf-8'>"
@ -619,5 +573,3 @@ if __name__=="__main__":
update_tags_list_for_each_hugo_themes() update_tags_list_for_each_hugo_themes()
update_tag_table() update_tag_table()
write_reports() write_reports()
"""
"""