trent
/
hugo_themes_report
mirror of https://github.com/TrentSPalmer/hugo_themes_report.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

adjust line length rank_hugo_themes.py

This commit is contained in:
Trent Palmer 2021-08-23 20:14:01 -07:00
parent 954db73a49
commit d21c2859d9
1 changed files with 155 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
rank_hugo_themes.py
View File

@ -94,7 +94,8 @@ else:
def get_hugo_themes_list(): def get_hugo_themes_list():
themes_list_url = f"https://raw.githubusercontent.com/{THEMESLISTREPO}/main/themes.txt" themes_list_url = "https://raw.githubusercontent.com/"
themes_list_url += f"{THEMESLISTREPO}/main/themes.txt"
response = get(themes_list_url) response = get(themes_list_url)
if response.status_code == 200: if response.status_code == 200:
@ -111,35 +112,52 @@ def get_hugo_themes_list():
def clean_up(): def clean_up():
themes_name_list = get_themes_name_list() themes_name_list = get_themes_name_list()
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
hugo_themes_name_list = [theme[0] for theme in session.query(Hugothemes.name).all()] hugo_themes_name_list = [
theme[0] for theme in session.query(Hugothemes.name).all()]
for theme_name in hugo_themes_name_list: for theme_name in hugo_themes_name_list:
if theme_name not in themes_name_list: if theme_name not in themes_name_list:
removed_theme = session.query(Hugothemes).filter_by(name=theme_name).first() removed_theme = session.query(
Hugothemes).filter_by(name=theme_name).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
gitlab_themes_name_list = get_gitlab_themes_name_list() gitlab_themes_name_list = get_gitlab_themes_name_list()
hugo_themes_from_gitlab_name_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] hugo_themes_from_gitlab_name_list = [
theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
for theme in hugo_themes_from_gitlab_name_list: for theme in hugo_themes_from_gitlab_name_list:
if theme not in gitlab_themes_name_list: if theme not in gitlab_themes_name_list:
removed_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).first() removed_theme = session.query(
Hugothemes_from_gitlab).filter_by(name=theme).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
def dedup_database(): def dedup_database():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
hugo_themes_sha_list = [theme[0] for theme in session.query(Hugothemes.commit_sha).all()] hugo_themes_sha_list = [
sha_list = [sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count(sha) > 1] theme[0] for theme in session.query(Hugothemes.commit_sha).all()]
sha_list = [
sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count(
sha
) > 1
]
for sha in sha_list: for sha in sha_list:
removed_theme = session.query(Hugothemes).filter_by(commit_sha=sha).first() removed_theme = session.query(
Hugothemes).filter_by(commit_sha=sha).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
hugo_themes_sha_list_from_gitlab = [theme[0] for theme in session.query(Hugothemes_from_gitlab.commit_sha).all()] hugo_themes_sha_list_from_gitlab = [theme[0] for theme in session.query(
sha_list_from_gitlab = [sha for sha in hugo_themes_sha_list_from_gitlab if hugo_themes_sha_list_from_gitlab.count(sha) > 1] Hugothemes_from_gitlab.commit_sha).all()]
sha_list_from_gitlab = [
sha for sha in hugo_themes_sha_list_from_gitlab
if hugo_themes_sha_list_from_gitlab.count(
sha
) > 1
]
for sha in sha_list_from_gitlab: for sha in sha_list_from_gitlab:
removed_theme = session.query(Hugothemes_from_gitlab).filter_by(commit_sha=sha).first() removed_theme = session.query(
Hugothemes_from_gitlab).filter_by(commit_sha=sha).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
@ -149,7 +167,8 @@ def parse_gitlab_hugo_themes_list():
gitlab_themes_list = get_gitlab_themes_list() gitlab_themes_list = get_gitlab_themes_list()
for theme in gitlab_themes_list: for theme in gitlab_themes_list:
theme_name = theme[11:] theme_name = theme[11:]
existing_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme_name).first() existing_theme = session.query(
Hugothemes_from_gitlab).filter_by(name=theme_name).first()
if existing_theme is None: if existing_theme is None:
session.add(Hugothemes_from_gitlab(name=theme_name, url=theme)) session.add(Hugothemes_from_gitlab(name=theme_name, url=theme))
session.commit() session.commit()
@ -163,7 +182,8 @@ def parse_hugo_themes_list():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
for theme in THEMESLIST: for theme in THEMESLIST:
theme_name = theme[11:] theme_name = theme[11:]
existing_theme = session.query(Hugothemes).filter_by(name=theme_name).first() existing_theme = session.query(
Hugothemes).filter_by(name=theme_name).first()
if existing_theme is None: if existing_theme is None:
session.add(Hugothemes(name=theme_name, url=theme)) session.add(Hugothemes(name=theme_name, url=theme))
session.commit() session.commit()
@ -175,17 +195,21 @@ def parse_hugo_themes_list():
def get_gitlab_project_ids(): def get_gitlab_project_ids():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = (theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()) themes = (theme[0] for theme in session.query(
Hugothemes_from_gitlab.name).all())
match = re.compile(r'(Project ID: )(\d{5,})') match = re.compile(r'(Project ID: )(\d{5,})')
for theme in themes: for theme in themes:
gitlab_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).one() gitlab_theme = session.query(
Hugothemes_from_gitlab).filter_by(name=theme).one()
if gitlab_theme.gitlab_id is None: if gitlab_theme.gitlab_id is None:
response = get(f"https://{gitlab_theme.url}") response = get(f"https://{gitlab_theme.url}")
if response.status_code == 200: if response.status_code == 200:
gitlab_theme.gitlab_id = match.search(response.text).group(2) gitlab_theme.gitlab_id = match.search(response.text).group(2)
session.commit() session.commit()
if response.status_code == 404: if response.status_code == 404:
print(response.status_code, get_gitlab_project_ids.__name__, theme) print(
response.status_code,
get_gitlab_project_ids.__name__, theme)
print(response.status_code, get_gitlab_project_ids.__name__) print(response.status_code, get_gitlab_project_ids.__name__)
@ -194,7 +218,8 @@ def get_commit_info_for_hugo_themes():
theme_names_from_github = get_github_themes_name_list() theme_names_from_github = get_github_themes_name_list()
for hugo_theme in theme_names_from_github: for hugo_theme in theme_names_from_github:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
api_call_url = f'https://api.github.com/repos/{theme.name}/commits/{theme.default_branch}' api_call_url = 'https://api.github.com/repos/'
api_call_url += f'{theme.name}/commits/{theme.default_branch}'
if theme.ETag is not None: if theme.ETag is not None:
headers['If-None-Match'] = theme.ETag headers['If-None-Match'] = theme.ETag
@ -213,14 +238,18 @@ def get_commit_info_for_hugo_themes():
theme.ETag = response.headers['ETag'].lstrip('W/') theme.ETag = response.headers['ETag'].lstrip('W/')
result = response.json() result = response.json()
theme.commit_date = result['commit']['author']['date'] theme.commit_date = result['commit']['author']['date']
theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) theme.commit_date_in_seconds = timegm(
strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ'))
theme.commit_sha = result['commit']['tree']['sha'] theme.commit_sha = result['commit']['tree']['sha']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_commit_info_for_hugo_themes.__name__) print(
response.status_code, get_commit_info_for_hugo_themes.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print(response.status_code, get_commit_info_for_hugo_themes.__name__, hugo_theme) print(
response.status_code,
get_commit_info_for_hugo_themes.__name__, hugo_theme)
print(response.status_code, get_commit_info_for_hugo_themes.__name__) print(response.status_code, get_commit_info_for_hugo_themes.__name__)
@ -229,23 +258,36 @@ def get_commit_info_for_hugo_themes_from_gitlab():
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
# match = re.compile(r'(\.\d{3})(Z$)') # match = re.compile(r'(\.\d{3})(Z$)')
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() theme = session.query(
api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}/repository/commits/{theme.default_branch}' Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
api_call_url = 'https://gitlab.com/api/v4/projects/'
api_call_url += f'{theme.gitlab_id}/repository/'
api_call_url += f'commits/{theme.default_branch}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
# theme.commit_date = (match.sub(r'\2', result['created_at']))[0:19] + 'Z' # theme.commit_date = (match.sub(r'\2', result[
# 'created_at']))[0:19] + 'Z'
theme.commit_date = result['created_at'][0:19] + 'Z' theme.commit_date = result['created_at'][0:19] + 'Z'
theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) theme.commit_date_in_seconds = timegm(
strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ'))
theme.commit_sha = result['id'] theme.commit_sha = result['id']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__) print(
response.status_code,
get_commit_info_for_hugo_themes_from_gitlab.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__, hugo_theme) print(
print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__) response.status_code,
get_commit_info_for_hugo_themes_from_gitlab.__name__,
hugo_theme
)
print(
response.status_code,
get_commit_info_for_hugo_themes_from_gitlab.__name__)
def get_repo_info_for_hugo_themes(): def get_repo_info_for_hugo_themes():
@ -275,7 +317,9 @@ def get_repo_info_for_hugo_themes():
print(response.status_code, get_repo_info_for_hugo_themes.__name__) print(response.status_code, get_repo_info_for_hugo_themes.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print(response.status_code, get_repo_info_for_hugo_themes.__name__, hugo_theme) print(
response.status_code,
get_repo_info_for_hugo_themes.__name__, hugo_theme)
print(response.status_code, get_repo_info_for_hugo_themes.__name__) print(response.status_code, get_repo_info_for_hugo_themes.__name__)
@ -283,7 +327,8 @@ def get_repo_info_for_hugo_themes_from_gitlab():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes_from_gitlab).filter_by(
name=hugo_theme).one()
api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}' api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
@ -294,11 +339,17 @@ def get_repo_info_for_hugo_themes_from_gitlab():
theme.default_branch = result['default_branch'] theme.default_branch = result['default_branch']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__) print(
response.status_code,
get_repo_info_for_hugo_themes_from_gitlab.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__, hugo_theme) print(
print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__) response.status_code,
get_repo_info_for_hugo_themes_from_gitlab.__name__, hugo_theme)
print(
response.status_code,
get_repo_info_for_hugo_themes_from_gitlab.__name__)
def get_theme_dot_toml_for_each_hugo_themes(): def get_theme_dot_toml_for_each_hugo_themes():
@ -310,7 +361,8 @@ def get_theme_dot_toml_for_each_hugo_themes():
theme_toml = 'wowchemy/theme.toml' theme_toml = 'wowchemy/theme.toml'
else: else:
theme_toml = 'theme.toml' theme_toml = 'theme.toml'
api_call_url = f"https://api.github.com/repos/{theme.name}/contents/{theme_toml}" api_call_url = "https://api.github.com/repos/"
api_call_url += f"{theme.name}/contents/{theme_toml}"
if theme.themes_toml_ETag is not None: if theme.themes_toml_ETag is not None:
headers['If-None-Match'] = theme.themes_toml_ETag headers['If-None-Match'] = theme.themes_toml_ETag
@ -328,51 +380,75 @@ def get_theme_dot_toml_for_each_hugo_themes():
theme.themes_toml_content = result['content'] theme.themes_toml_content = result['content']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__) print(
response.status_code,
get_theme_dot_toml_for_each_hugo_themes.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme) print(
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__) response.status_code,
get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme)
print(
response.status_code,
get_theme_dot_toml_for_each_hugo_themes.__name__)
def get_theme_dot_toml_for_each_hugo_themes_from_gitlab(): def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() theme = session.query(
api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}/repository/files/theme.toml?ref={theme.default_branch}' Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
api_call_url = 'https://gitlab.com/api/v4/projects/'
api_call_url += f'{theme.gitlab_id}/repository/files/'
api_call_url += f'theme.toml?ref={theme.default_branch}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
theme.themes_toml_content = result['content'] theme.themes_toml_content = result['content']
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) print(
response.status_code,
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__, hugo_theme) print(
print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) response.status_code,
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__,
hugo_theme)
print(
response.status_code,
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
def coalesce_themes(): def coalesce_themes():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
htfgitlab = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() htfgitlab = session.query(
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
theme = session.query(Hugothemes).filter_by(name=hugo_theme).first() theme = session.query(Hugothemes).filter_by(name=hugo_theme).first()
if theme is None: if theme is None:
session.add(Hugothemes( session.add(Hugothemes(
name=hugo_theme, url=htfgitlab.url, commit_sha=htfgitlab.commit_sha, name=hugo_theme, url=htfgitlab.url,
commit_date=htfgitlab.commit_date, commit_date_in_seconds=htfgitlab.commit_date_in_seconds, commit_sha=htfgitlab.commit_sha,
stargazers_count=htfgitlab.star_count, themes_toml_content=htfgitlab.themes_toml_content commit_date=htfgitlab.commit_date,
commit_date_in_seconds=htfgitlab.commit_date_in_seconds,
stargazers_count=htfgitlab.star_count,
themes_toml_content=htfgitlab.themes_toml_content,
)) ))
else: else:
if theme.url != htfgitlab.url: theme.url = htfgitlab.url if theme.url != htfgitlab.url: theme.url = htfgitlab.url
if theme.commit_sha != htfgitlab.commit_sha: theme.commit_sha = htfgitlab.commit_sha if theme.commit_sha != htfgitlab.commit_sha:
if theme.commit_date != htfgitlab.commit_date: theme.commit_date = htfgitlab.commit_date theme.commit_sha = htfgitlab.commit_sha
if theme.commit_date_in_seconds != htfgitlab.commit_date_in_seconds: if theme.commit_date != htfgitlab.commit_date:
theme.commit_date_in_seconds = htfgitlab.commit_date_in_seconds theme.commit_date = htfgitlab.commit_date
if theme.stargazers_count != htfgitlab.star_count: theme.stargazers_count = htfgitlab.star_count htfgitlab_cdis = htfgitlab.commit_date_in_seconds
if theme.commit_date_in_seconds != htfgitlab_cdis:
theme.commit_date_in_seconds = htfgitlab_cdis
if theme.stargazers_count != htfgitlab.star_count:
theme.stargazers_count = htfgitlab.star_count
theme.themes_toml_content = htfgitlab.themes_toml_content theme.themes_toml_content = htfgitlab.themes_toml_content
session.commit() session.commit()
@ -405,10 +481,13 @@ def parse_themes_toml_for_each_hugo_themes():
if 'tags' in theme_toml: if 'tags' in theme_toml:
if len(theme_toml['tags']) > 0: if len(theme_toml['tags']) > 0:
corrected_tags = get_corrected_tags(theme_toml['tags']) corrected_tags = get_corrected_tags(theme_toml['tags'])
theme_tags = [tag.lower() for tag in corrected_tags if len(tag) > 1] theme_tags = [
if theme.num_tags != len(theme_tags): theme.num_tags = len(theme_tags) tag.lower() for tag in corrected_tags if len(tag) > 1]
if theme.num_tags != len(theme_tags):
theme.num_tags = len(theme_tags)
if theme.num_tags > 0: if theme.num_tags > 0:
if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags) if theme.tags_list != str(theme_tags):
theme.tags_list = str(theme_tags)
else: else:
if theme.tags_list is not None: theme.tags_list = None if theme.tags_list is not None: theme.tags_list = None
else: else:
@ -419,14 +498,20 @@ def parse_themes_toml_for_each_hugo_themes():
if theme.num_tags != 0: theme.num_tags = 0 if theme.num_tags != 0: theme.num_tags = 0
if 'features' in theme_toml: if 'features' in theme_toml:
if len(theme_toml['features']) > 0: if len(theme_toml['features']) > 0:
theme_features = [feature.lower() for feature in theme_toml['features'] if len(feature) > 1] theme_features = [
if theme.num_features != len(theme_features): theme.num_features = len(theme_features) feature.lower() for feature in theme_toml[
'features'] if len(feature) > 1]
if theme.num_features != len(theme_features):
theme.num_features = len(theme_features)
if theme.num_features > 0: if theme.num_features > 0:
if theme.features_list != str(theme_features): theme.features_list = str(theme_features) if theme.features_list != str(theme_features):
theme.features_list = str(theme_features)
else: else:
if theme.features_list is not None: theme.features_list = None if theme.features_list is not None:
theme.features_list = None
else: else:
if theme.features_list is not None: theme.features_list = None if theme.features_list is not None:
theme.features_list = None
if theme.num_features != 0: theme.num_features = 0 if theme.num_features != 0: theme.num_features = 0
else: else:
if theme.features_list is not None: theme.features_list = None if theme.features_list is not None: theme.features_list = None
@ -474,12 +559,17 @@ def generate_report():
'url': f'https://{theme.url}', 'url': f'https://{theme.url}',
'short_name': theme.name.split('/')[1], 'short_name': theme.name.split('/')[1],
'num_stars': theme.stargazers_count, 'num_stars': theme.stargazers_count,
'tags': literal_eval(theme.tags_list) if theme.tags_list is not None else [], 'tags': literal_eval(
'features': literal_eval(theme.features_list) if theme.features_list is not None else [], theme.tags_list) if theme.tags_list is not None else [],
'license': theme.theme_license if theme.theme_license is not None else '', 'features': literal_eval(
theme.features_list
) if theme.features_list is not None else [],
'license': theme.theme_license if
theme.theme_license is not None else '',
'min_ver': theme.min_ver if theme.min_ver is not None else '', 'min_ver': theme.min_ver if theme.min_ver is not None else '',
'desc': theme.desc if theme.desc is not None else '', 'desc': theme.desc if theme.desc is not None else '',
'cname': theme.cname if theme.cname is not None else theme.name.split('/')[1], 'cname': theme.cname if
theme.cname is not None else theme.name.split('/')[1],
} for theme in session.query(Hugothemes).all() } for theme in session.query(Hugothemes).all()
] ]
output = template.render(themes=hugo_themes) output = template.render(themes=hugo_themes)
@ -492,7 +582,9 @@ if __name__ == "__main__":
''' '''
or test with or test with
` `
python3 -c'import rank_hugo_themes; rank_hugo_themes.parse_themes_toml_for_each_hugo_themes() ; rank_hugo_themes.generate_report()' python3 -c'import rank_hugo_themes; rank_hugo_themes
.parse_themes_toml_for_each_hugo_themes()
; rank_hugo_themes.generate_report()'
` `
''' '''
dedup_database() dedup_database()