trent/hugo_themes_report
1
0
Fork 0
mirror of https://github.com/TrentSPalmer/hugo_themes_report.git synced 2025-04-28 00:02:48 -07:00
Code Issues Projects Releases Wiki Activity

major rewrite rank_hugo_themes.py

Browse Source
This commit is contained in:
Trent Palmer 2021-08-09 15:39:43 -07:00
parent cb8c491a3e
commit 46b919ff50
1 changed files with 264 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

350
rank_hugo_themes.py
View File

@ -6,7 +6,7 @@ import toml
from calendar import timegm from calendar import timegm
from time import strptime from time import strptime
from requests import get from requests import get
from json import loads as json_loads # from json import loads as json_loads
from sys import argv as sys_argv from sys import argv as sys_argv
from base64 import b64decode from base64 import b64decode
from ast import literal_eval from ast import literal_eval
@ -43,12 +43,16 @@ class Hugothemes_from_gitlab(Base):
commit_date = Column(TEXT) commit_date = Column(TEXT)
star_count = Column(Integer) star_count = Column(Integer)
themes_toml_content = Column(TEXT) themes_toml_content = Column(TEXT)
default_branch = Column(TEXT)
def __repr__(self): def __repr__(self):
repr_string = "<(name = '%s', url = '%s', commit_sha = '%s', gitlab_id = '%s', commit_date_in_seconds = '%s'" repr_string = "<(name = '%s', url = '%s', commit_sha = '%s', gitlab_id = '%s', commit_date_in_seconds = '%s'"
repr_string += ", commit_date = '%s', star_count = '%s', themes_toml_content = '%s')>" repr_string += ", commit_date = '%s', star_count = '%s', themes_toml_content = '%s', default_branch = '%s')>"
repr_values = (self.name,self.commit_sha,self.gitlab_id,self.commit_date_in_seconds,self.commit_date, repr_values = (
self.star_count,self.themes_toml_content) self.name, self.commit_sha, self.gitlab_id,
self.commit_date_in_seconds, self.commit_date,
self.star_count, self.themes_toml_content, self.default_branch
)
return repr_string % repr_values return repr_string % repr_values
@ -58,7 +62,6 @@ class Hugothemes(Base):
name = Column(VARCHAR, primary_key=True) name = Column(VARCHAR, primary_key=True)
ETag = Column(TEXT) ETag = Column(TEXT)
url = Column(TEXT) url = Column(TEXT)
jsondump = deferred(Column(TEXT))
commit_sha = Column(TEXT) commit_sha = Column(TEXT)
commit_date = Column(TEXT) commit_date = Column(TEXT)
commit_date_in_seconds = Column(Integer) commit_date_in_seconds = Column(Integer)
@ -68,19 +71,41 @@ class Hugothemes(Base):
themes_toml_content = deferred(Column(TEXT)) themes_toml_content = deferred(Column(TEXT))
tags_list = Column(TEXT) tags_list = Column(TEXT)
num_tags = Column(Integer) num_tags = Column(Integer)
dot_gitmodules_content = Column(TEXT) default_branch = Column(TEXT)
def __repr__(self): def __repr__(self):
repr_string = "<(name = '%s', ETag = '%s', url = '%s', jsondump = '%s', commit_sha = '%s', commit_date = '%s'" repr_string = "<(name = '%s', ETag = '%s', url = '%s', commit_sha = '%s', commit_date = '%s'"
repr_string += ", commit_date_in_seconds = '%s', repo_ETag = '%s', stargazers_count = '%s', themes_toml_ETag = '%s'" repr_string += ", commit_date_in_seconds = '%s', repo_ETag = '%s', stargazers_count = '%s', themes_toml_ETag = '%s'"
repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', dot_gitmodule_content = '%s')>" repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', default_branch = '%s')>"
repr_values = (self.name,self.ETag,self.url,self.jsondump,self.commit_sha,self.commit_date,self.commit_date_in_seconds, repr_values = (
self.repo_ETag,self.stargazers_count,self.themes_toml_ETag,self.themes_toml_content,self.tags_list, self.name, self.ETag, self.url,
self.num_tags,self.dot_gitmodules_content) self.commit_sha, self.commit_date, self.commit_date_in_seconds,
self.repo_ETag, self.stargazers_count, self.themes_toml_ETag,
self.themes_toml_content, self.tags_list,
self.num_tags, self.default_branch
)
return repr_string % repr_values return repr_string % repr_values
THEMESLISTREPO = 'gohugoio/hugoThemes' OLDTHEMESLISTREPO = 'gohugoio/hugoThemes'
THEMESLISTREPO = 'gohugoio/hugoThemesSiteBuilder'
THEMESLIST = []
def get_themes_name_list():
return [x[11:] for x in THEMESLIST]
def get_gitlab_themes_list():
return [x for x in THEMESLIST if x[0:10] == 'gitlab.com']
def get_gitlab_themes_name_list():
return [x[11:] for x in THEMESLIST if x[0:10] == 'gitlab.com']
def get_github_themes_name_list():
return [x[11:] for x in THEMESLIST if x[0:10] == 'github.com']
if len(sys_argv) == 2: if len(sys_argv) == 2:
@ -90,144 +115,62 @@ else:
def get_hugo_themes_list(): def get_hugo_themes_list():
session = sessionmaker(bind=engine)() themes_list_url = f"https://raw.githubusercontent.com/{THEMESLISTREPO}/main/themes.txt"
themes_list_repo = session.query(Hugothemes).filter_by(name=THEMESLISTREPO).first() response = get(themes_list_url)
api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents"
if themes_list_repo.url != api_call_url: themes_list_repo.url = api_call_url
if themes_list_repo.ETag != None:
headers['If-None-Match'] = themes_list_repo.ETag
else:
if 'If-None-Match' in headers: del headers['If-None-Match']
if len(headers) == 0:
response = get(api_call_url)
else:
response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
themes_list_repo.ETag = response.headers['ETag'].lstrip('W/') for x in response.text.splitlines():
themes_list_repo.jsondump = response.text if (x[0:10] == 'gitlab.com' or x[0:10] == 'github.com'):
session.commit() THEMESLIST.append(x)
elif response.status_code == 403:
print(response.status_code, get_hugo_themes_list.__name__) print(response.status_code, get_hugo_themes_list.__name__)
session.commit()
write_reports()
quit()
print(response.status_code,get_hugo_themes_list.__name__)
def update_hugo_themes_submodule_list():
session = sessionmaker(bind=engine)()
themes_list_repo = session.query(Hugothemes).filter_by(name=THEMESLISTREPO).first()
themes_json = json_loads(themes_list_repo.jsondump)
if themes_list_repo.dot_gitmodules_content:
dot_gitmodule_json = json_loads(themes_list_repo.dot_gitmodules_content)
for theme in themes_json:
if theme['name'] == '.gitmodules':
new_gitmodules_sha = theme['sha']
break
if (not dot_gitmodule_json) or (new_gitmodules_sha != dot_gitmodule_json['sha']):
api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents/.gitmodules"
response = get(api_call_url)
if response.status_code == 200:
if themes_list_repo.dot_gitmodules_content != response.text:
themes_list_repo.dot_gitmodules_content = response.text
session.commit()
print(response.status_code,update_hugo_themes_submodule_list.__name__)
def get_hugo_themes_submodule_list():
session = sessionmaker(bind=engine)()
dot_gitmodule_json_string = session.query(Hugothemes.dot_gitmodules_content).filter_by(name=THEMESLISTREPO).first()
dot_gitmodule_json = json_loads(dot_gitmodule_json_string[0])
dot_gitmodule_content = b64decode(dot_gitmodule_json['content']).decode('utf-8').replace('\n\n','\n').split('\n')
submodules = []
for line in range(len(dot_gitmodule_content[:-1])):
if 'submodule' in dot_gitmodule_content[line]:
name = re.sub(r'(^\[submodule "|"\]$)','',dot_gitmodule_content[line])
if 'path = ' in dot_gitmodule_content[line+1]:
path = re.match(r'^.*(path = )(.*)$',dot_gitmodule_content[line+1]).group(2)
if 'url = ' in dot_gitmodule_content[line+2]:
url = re.match(r'^.*(url = )(.*)$',dot_gitmodule_content[line+2]).group(2)
submodules.append((name,path,url))
return submodules
def clean_up(): def clean_up():
new_name_list = set() themes_name_list = get_themes_name_list()
submodules = get_hugo_themes_submodule_list()
for submodule in submodules:
submodule_name = re.sub(r'(^https://github.com/|^https://gitlab.com/|\.git$)','',submodule[2])
new_name_list.add(submodule_name)
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
old_name_list = [theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all()] hugo_themes_name_list = [theme[0] for theme in session.query(Hugothemes.name).all()]
for theme_name in hugo_themes_name_list:
for name in old_name_list: if theme_name not in themes_name_list:
if name not in new_name_list: removed_theme = session.query(Hugothemes).filter_by(name=theme_name).first()
removed_theme = session.query(Hugothemes).filter_by(name=name).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit()
themes_json_string = session.query(Hugothemes.jsondump).filter_by(name=THEMESLISTREPO).first() gitlab_themes_name_list = get_gitlab_themes_name_list()
themes_json = json_loads(themes_json_string[0]) hugo_themes_from_gitlab_name_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
new_name_list = set() for theme in hugo_themes_from_gitlab_name_list:
for theme in themes_json: if theme not in gitlab_themes_name_list:
if theme['git_url']: removed_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).first()
if 'gohugoio' not in theme['git_url']: session.delete(removed_theme)
split_html_url = theme['html_url'].split('/')
new_short_name = split_html_url[3]+'/'+split_html_url[4]
new_name_list.add(new_short_name)
for name in old_name_list:
if name not in new_name_list:
removed_theme = session.query(Hugothemes).filter_by(name=name).first()
if removed_theme != None: session.delete(removed_theme)
session.commit() session.commit()
def parse_submodules_from_gitlab(): def parse_gitlab_hugo_themes_list():
submodules = get_hugo_themes_submodule_list()
temp_submodules_list = []
for submodule in submodules:
if 'gitlab' in submodule[2]:
temp_submodules_list.append(submodule)
if (len(temp_submodules_list) > 0):
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes_json_string = session.query(Hugothemes.jsondump).filter_by(name=THEMESLISTREPO).first() gitlab_themes_list = get_gitlab_themes_list()
themes_json = json_loads(themes_json_string[0]) for theme in gitlab_themes_list:
submodules_list = [] theme_name = theme[11:]
for submodule in temp_submodules_list: existing_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme_name).first()
for theme in themes_json: if existing_theme is None:
if submodule[0] == theme['name']: session.add(Hugothemes_from_gitlab(name=theme_name, url=theme))
if (theme['html_url'] == None) or (theme['git_url'] == None): session.commit()
submodules_list.append((submodule[0],submodule[1],submodule[2],theme['sha']))
if (len(submodules_list) > 0):
for submodule in submodules_list:
url = re.sub(r'\.git$','',submodule[2])
theme_name = url[19:]
theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme_name).first()
if theme == None:
session.add(Hugothemes_from_gitlab(name=theme_name,url=url,commit_sha=submodule[3]))
else: else:
if theme.url != url: theme.url = url if existing_theme.url != theme:
if theme.commit_sha != submodule[3]: theme.commit_sha = submodule[3] existing_theme.url = theme
session.commit() session.commit()
def parse_hugo_themes_list(): def parse_hugo_themes_list():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes_json_string = session.query(Hugothemes.jsondump).filter_by(name=THEMESLISTREPO).first() for theme in THEMESLIST:
themes_json = json_loads(themes_json_string[0]) theme_name = theme[11:]
for x in themes_json: existing_theme = session.query(Hugothemes).filter_by(name=theme_name).first()
if x['git_url']: if existing_theme is None:
if 'gohugoio' not in x['git_url']: session.add(Hugothemes(name=theme_name, url=theme))
theme_git_url = x['html_url'][:-46] session.commit()
theme_git_name = theme_git_url[19:]
theme = session.query(Hugothemes).filter_by(name=theme_git_name).first()
if theme == None:
session.add(Hugothemes(name=theme_git_name,url=theme_git_url,commit_sha=x['sha']))
else: else:
if theme.url != theme_git_url: theme.url = theme_git_url if existing_theme.url != theme:
if theme.commit_sha != x['sha']: theme.commit_sha = x['sha'] existing_theme.url = theme
session.commit() session.commit()
@ -237,24 +180,24 @@ def get_gitlab_project_ids():
match = re.compile(r'(Project ID: )(\d{5,})') match = re.compile(r'(Project ID: )(\d{5,})')
for theme in themes: for theme in themes:
gitlab_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).one() gitlab_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).one()
if gitlab_theme.gitlab_id == None: if gitlab_theme.gitlab_id is None:
response = get(gitlab_theme.url) response = get(f"https://{gitlab_theme.url}")
if response.status_code == 200:
gitlab_theme.gitlab_id = match.search(response.text).group(2) gitlab_theme.gitlab_id = match.search(response.text).group(2)
session.commit() session.commit()
return True if response.status_code == 404:
print(response.status_code, get_gitlab_project_ids.__name__, theme)
print(response.status_code, get_gitlab_project_ids.__name__)
def get_commit_info_for_hugo_themes(): def get_commit_info_for_hugo_themes():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
non_github_theme_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] theme_names_from_github = get_github_themes_name_list()
themes = set(theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all()) for hugo_theme in theme_names_from_github:
for theme in non_github_theme_list:
themes -= {theme}
for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
api_call_url = 'https://api.github.com/repos/' + theme.name + '/commits/' + theme.commit_sha api_call_url = f'https://api.github.com/repos/{theme.name}/commits/{theme.default_branch}'
if theme.ETag != None: if theme.ETag is not None:
headers['If-None-Match'] = theme.ETag headers['If-None-Match'] = theme.ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers:
@ -264,52 +207,58 @@ def get_commit_info_for_hugo_themes():
response = get(api_call_url) response = get(api_call_url)
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 422:
print(hugo_theme)
quit()
if response.status_code == 200: if response.status_code == 200:
theme.ETag = response.headers['ETag'].lstrip('W/') theme.ETag = response.headers['ETag'].lstrip('W/')
result = response.json() result = response.json()
theme.commit_date = result['commit']['author']['date'] theme.commit_date = result['commit']['author']['date']
theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ'))
theme.commit_sha = result['commit']['tree']['sha']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_commit_info_for_hugo_themes.__name__) print(response.status_code, get_commit_info_for_hugo_themes.__name__)
write_reports() # write_reports()
quit() quit()
elif response.status_code == 404:
print(response.status_code, get_commit_info_for_hugo_themes.__name__, hugo_theme)
print(response.status_code, get_commit_info_for_hugo_themes.__name__) print(response.status_code, get_commit_info_for_hugo_themes.__name__)
def get_commit_info_for_hugo_themes_from_gitlab(): def get_commit_info_for_hugo_themes_from_gitlab():
if get_gitlab_project_ids():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] theme_names_from_gitlab = get_gitlab_themes_name_list()
match = re.compile(r'(\.\d{3})(Z$)') # match = re.compile(r'(\.\d{3})(Z$)')
for hugo_theme in themes: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
api_call_url = 'https://gitlab.com/api/v4/projects/' + theme.gitlab_id + '/repository/commits/' + theme.commit_sha api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}/repository/commits/{theme.default_branch}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
theme.commit_date = (match.sub(r'\2',result['created_at']))[0:19] + 'Z' # theme.commit_date = (match.sub(r'\2', result['created_at']))[0:19] + 'Z'
theme.commit_date = result['created_at'][0:19] + 'Z'
theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ'))
theme.commit_sha = result['id']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__) print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__)
write_reports() # write_reports()
quit() quit()
elif response.status_code == 404:
print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__, hugo_theme)
print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__) print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__)
def get_stargazer_count_for_hugo_themes(): def get_stargazer_count_for_hugo_themes():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
non_github_theme_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] theme_names_from_github = get_github_themes_name_list()
themes = set(theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all()) for hugo_theme in theme_names_from_github:
for theme in non_github_theme_list:
themes -= {theme}
for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
api_call_url = 'https://api.github.com/repos/' + theme.name api_call_url = 'https://api.github.com/repos/' + theme.name
if theme.repo_ETag != None: if theme.repo_ETag is not None:
headers['If-None-Match'] = theme.repo_ETag headers['If-None-Match'] = theme.repo_ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers:
@ -323,44 +272,52 @@ def get_stargazer_count_for_hugo_themes():
theme.repo_ETag = response.headers['ETag'].lstrip('W/') theme.repo_ETag = response.headers['ETag'].lstrip('W/')
result = response.json() result = response.json()
theme.stargazers_count = result['stargazers_count'] theme.stargazers_count = result['stargazers_count']
theme.default_branch = result['default_branch']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_stargazer_count_for_hugo_themes.__name__) print(response.status_code, get_stargazer_count_for_hugo_themes.__name__)
write_reports() # write_reports()
quit() quit()
elif response.status_code == 404:
print(response.status_code, get_stargazer_count_for_hugo_themes.__name__, hugo_theme)
print(response.status_code, get_stargazer_count_for_hugo_themes.__name__) print(response.status_code, get_stargazer_count_for_hugo_themes.__name__)
def get_stargazer_count_for_hugo_themes_from_gitlab(): def get_stargazer_count_for_hugo_themes_from_gitlab():
if get_gitlab_project_ids():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = [(theme,gitlab_id) for theme,gitlab_id in session.query(Hugothemes_from_gitlab.name,Hugothemes_from_gitlab.gitlab_id).all()] theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in themes: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme[0]).one() theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
api_call_url = 'https://gitlab.com/api/v4/projects/' + hugo_theme[1] api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
if theme.star_count != result['star_count']: if theme.star_count != result['star_count']:
theme.star_count = result['star_count'] theme.star_count = result['star_count']
if theme.default_branch != result['default_branch']:
theme.default_branch = result['default_branch']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_stargazer_count_for_hugo_themes_from_gitlab.__name__) print(response.status_code, get_stargazer_count_for_hugo_themes_from_gitlab.__name__)
write_reports() # write_reports()
quit() quit()
elif response.status_code == 404:
print(response.status_code, get_stargazer_count_for_hugo_themes_from_gitlab.__name__, hugo_theme)
print(response.status_code, get_stargazer_count_for_hugo_themes_from_gitlab.__name__) print(response.status_code, get_stargazer_count_for_hugo_themes_from_gitlab.__name__)
def get_theme_dot_toml_for_each_hugo_themes(): def get_theme_dot_toml_for_each_hugo_themes():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
non_github_theme_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] theme_names_from_github = get_github_themes_name_list()
themes = set(theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name!=THEMESLISTREPO).all()) for hugo_theme in theme_names_from_github:
for theme in non_github_theme_list: themes -= {theme}
for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
api_call_url = "https://api.github.com/repos/"+theme.name+"/contents/theme.toml" if theme.name == 'gcushen/hugo-academic':
theme_toml = 'wowchemy/theme.toml'
else:
theme_toml = 'theme.toml'
api_call_url = f"https://api.github.com/repos/{theme.name}/contents/{theme_toml}"
if theme.themes_toml_ETag != None: if theme.themes_toml_ETag is not None:
headers['If-None-Match'] = theme.themes_toml_ETag headers['If-None-Match'] = theme.themes_toml_ETag
else: else:
if 'If-None-Match' in headers: if 'If-None-Match' in headers:
@ -377,18 +334,19 @@ def get_theme_dot_toml_for_each_hugo_themes():
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__) print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__)
write_reports() # write_reports()
quit() quit()
elif response.status_code == 404:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme)
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__) print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__)
def get_theme_dot_toml_for_each_hugo_themes_from_gitlab(): def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
if get_gitlab_project_ids():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = [(theme,gitlab_id) for theme,gitlab_id in session.query(Hugothemes_from_gitlab.name,Hugothemes_from_gitlab.gitlab_id).all()] theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in themes: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme[0]).one() theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
api_call_url = 'https://gitlab.com/api/v4/projects/' + hugo_theme[1] + '/repository/files/theme.toml?ref=master' api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}/repository/files/theme.toml?ref={theme.default_branch}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
@ -396,22 +354,25 @@ def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
write_reports() # write_reports()
quit() quit()
elif response.status_code == 404:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__, hugo_theme)
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
def coalesce_themes(): def coalesce_themes():
if get_gitlab_project_ids():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in themes: for hugo_theme in theme_names_from_gitlab:
htfgitlab = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() htfgitlab = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
theme = session.query(Hugothemes).filter_by(name=hugo_theme).first() theme = session.query(Hugothemes).filter_by(name=hugo_theme).first()
if theme == None: if theme is None:
session.add(Hugothemes(name=hugo_theme,url=htfgitlab.url,commit_sha=htfgitlab.commit_sha, session.add(Hugothemes(
name=hugo_theme, url=htfgitlab.url, commit_sha=htfgitlab.commit_sha,
commit_date=htfgitlab.commit_date, commit_date_in_seconds=htfgitlab.commit_date_in_seconds, commit_date=htfgitlab.commit_date, commit_date_in_seconds=htfgitlab.commit_date_in_seconds,
stargazers_count=htfgitlab.star_count,themes_toml_content=htfgitlab.themes_toml_content)) stargazers_count=htfgitlab.star_count, themes_toml_content=htfgitlab.themes_toml_content
))
else: else:
if theme.url != htfgitlab.url: theme.url = htfgitlab.url if theme.url != htfgitlab.url: theme.url = htfgitlab.url
if theme.commit_sha != htfgitlab.commit_sha: theme.commit_sha = htfgitlab.commit_sha if theme.commit_sha != htfgitlab.commit_sha: theme.commit_sha = htfgitlab.commit_sha
@ -429,7 +390,7 @@ def update_tags_list_for_each_hugo_themes():
match = re.compile(r'\s(\d+\.\d+\.\d+)\s') match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
for hugo_theme in themes: for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
if theme.themes_toml_content != None: if theme.themes_toml_content is not None:
content = b64decode(theme.themes_toml_content).decode('utf-8') content = b64decode(theme.themes_toml_content).decode('utf-8')
# put quotes around any unquoted double-dotted version numbers # put quotes around any unquoted double-dotted version numbers
# (and add a newline afterwards) # (and add a newline afterwards)
@ -442,26 +403,26 @@ def update_tags_list_for_each_hugo_themes():
if theme.num_tags > 0: if theme.num_tags > 0:
if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags) if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags)
else: else:
if theme.tags_list != None: theme.tags_list = None if theme.tags_list is not None: theme.tags_list = None
else: else:
if theme.tags_list != None: theme.tags_list = None if theme.tags_list is not None: theme.tags_list = None
if theme.num_tags != 0: theme.num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
else: else:
if theme.tags_list != None: theme.tags_list = None if theme.tags_list is not None: theme.tags_list = None
if theme.num_tags != 0: theme.num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
else: else:
if theme.tags_list != None: theme.tags_list = None if theme.tags_list is not None: theme.tags_list = None
if theme.num_tags != 0: theme.num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
session.commit() session.commit()
def update_tag_table(): def update_tag_table():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = [(theme,tags_list) for theme,tags_list in session.query(Hugothemes.name,Hugothemes.tags_list) themes = [(theme, tags_list) for theme, tags_list in session.query(
.filter(Hugothemes.name!=THEMESLISTREPO).all()] Hugothemes.name, Hugothemes.tags_list).filter(Hugothemes.name != THEMESLISTREPO).all()]
tags_list = set() tags_list = set()
for theme in themes: for theme in themes:
if theme[1] != None: if theme[1] is not None:
tags = literal_eval(theme[1]) tags = literal_eval(theme[1])
for tag in tags: for tag in tags:
if len(tag) > 0: if len(tag) > 0:
@ -470,12 +431,12 @@ def update_tag_table():
for hugo_tag in tags_list: for hugo_tag in tags_list:
theme_list = [] theme_list = []
for theme in themes: for theme in themes:
if theme[1] != None: if theme[1] is not None:
tags = literal_eval(theme[1]) tags = literal_eval(theme[1])
if hugo_tag in tags: if hugo_tag in tags:
theme_list.append(theme[0]) theme_list.append(theme[0])
tag = session.query(Tags).filter_by(tag=hugo_tag).first() tag = session.query(Tags).filter_by(tag=hugo_tag).first()
if tag == None: if tag is None:
session.add(Tags(tag=hugo_tag, theme_list=str(theme_list), num_themes=len(theme_list))) session.add(Tags(tag=hugo_tag, theme_list=str(theme_list), num_themes=len(theme_list)))
else: else:
theme_list, num_themes = str(theme_list), len(theme_list) theme_list, num_themes = str(theme_list), len(theme_list)
@ -560,14 +521,15 @@ def write_reports():
if __name__ == "__main__": if __name__ == "__main__":
get_hugo_themes_list() get_hugo_themes_list()
update_hugo_themes_submodule_list() if len(THEMESLIST) > 300:
clean_up() clean_up()
parse_submodules_from_gitlab()
parse_hugo_themes_list() parse_hugo_themes_list()
get_commit_info_for_hugo_themes() parse_gitlab_hugo_themes_list()
get_commit_info_for_hugo_themes_from_gitlab() get_gitlab_project_ids()
get_stargazer_count_for_hugo_themes() get_stargazer_count_for_hugo_themes()
get_stargazer_count_for_hugo_themes_from_gitlab() get_stargazer_count_for_hugo_themes_from_gitlab()
get_commit_info_for_hugo_themes()
get_commit_info_for_hugo_themes_from_gitlab()
get_theme_dot_toml_for_each_hugo_themes() get_theme_dot_toml_for_each_hugo_themes()
get_theme_dot_toml_for_each_hugo_themes_from_gitlab() get_theme_dot_toml_for_each_hugo_themes_from_gitlab()
coalesce_themes() coalesce_themes()