trent
/
hugo_themes_report
mirror of https://github.com/TrentSPalmer/hugo_themes_report.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update get_corrected_theme_name() in rank_hugo_themes.py

This commit is contained in:
Trent Palmer 2022-05-08 13:55:18 -07:00
parent 4d4e378031
commit d9505fcea6
1 changed files with 195 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

384
rank_hugo_themes.py
View File

@ -1,29 +1,29 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# rank_hugo_themes.py # rank_hugo_themes.py
from jinja2 import Environment, FileSystemLoader
import re import re
import toml
from calendar import timegm
from time import strptime
from requests import get
from sys import argv as sys_argv
from base64 import b64decode
from ast import literal_eval from ast import literal_eval
from base64 import b64decode
from calendar import timegm
from sys import argv as sys_argv
from time import strptime
from sqlalchemy import create_engine, Column, Integer, VARCHAR, TEXT import toml
from jinja2 import Environment, FileSystemLoader
from requests import get
from sqlalchemy import TEXT, VARCHAR, Column, Integer, create_engine
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import deferred, sessionmaker from sqlalchemy.orm import deferred, sessionmaker
engine = create_engine('sqlite:///hugothemes.db', echo=False) engine = create_engine("sqlite:///hugothemes.db", echo=False)
Base = declarative_base() Base = declarative_base()
file_loader = FileSystemLoader('templates') file_loader = FileSystemLoader("templates")
env = Environment(loader=file_loader) env = Environment(loader=file_loader)
template = env.get_template('base.html') template = env.get_template("base.html")
class Hugothemes_from_gitlab(Base): class Hugothemes_from_gitlab(Base):
__tablename__ = 'hugothemes_from_gitlab' __tablename__ = "hugothemes_from_gitlab"
name = Column(VARCHAR, primary_key=True) name = Column(VARCHAR, primary_key=True)
url = Column(TEXT) url = Column(TEXT)
@ -40,7 +40,7 @@ class Hugothemes_from_gitlab(Base):
class Hugothemes(Base): class Hugothemes(Base):
__tablename__ = 'hugothemes' __tablename__ = "hugothemes"
name = Column(VARCHAR, primary_key=True) name = Column(VARCHAR, primary_key=True)
ETag = Column(TEXT) ETag = Column(TEXT)
@ -66,8 +66,8 @@ class Hugothemes(Base):
return f"<Hugothemes(name={self.name})>" return f"<Hugothemes(name={self.name})>"
OLDTHEMESLISTREPO = 'gohugoio/hugoThemes' OLDTHEMESLISTREPO = "gohugoio/hugoThemes"
THEMESLISTREPO = 'gohugoio/hugoThemesSiteBuilder' THEMESLISTREPO = "gohugoio/hugoThemesSiteBuilder"
THEMESLIST = [] THEMESLIST = []
@ -76,19 +76,19 @@ def get_themes_name_list():
def get_gitlab_themes_list(): def get_gitlab_themes_list():
return [x for x in THEMESLIST if x[0:10] == 'gitlab.com'] return [x for x in THEMESLIST if x[0:10] == "gitlab.com"]
def get_gitlab_themes_name_list(): def get_gitlab_themes_name_list():
return [x[11:] for x in THEMESLIST if x[0:10] == 'gitlab.com'] return [x[11:] for x in THEMESLIST if x[0:10] == "gitlab.com"]
def get_github_themes_name_list(): def get_github_themes_name_list():
return [x[11:] for x in THEMESLIST if x[0:10] == 'github.com'] return [x[11:] for x in THEMESLIST if x[0:10] == "github.com"]
if len(sys_argv) == 2: if len(sys_argv) == 2:
headers = {'Authorization': 'token ' + sys_argv[1]} headers = {"Authorization": "token " + sys_argv[1]}
else: else:
headers = {} headers = {}
@ -101,7 +101,7 @@ def get_hugo_themes_list():
if response.status_code == 200: if response.status_code == 200:
lower_case_themes_list = [] lower_case_themes_list = []
for x in response.text.splitlines(): for x in response.text.splitlines():
if (x[0:10] == 'gitlab.com' or x[0:10] == 'github.com'): if x[0:10] == "gitlab.com" or x[0:10] == "github.com":
if x.lower() not in lower_case_themes_list: if x.lower() not in lower_case_themes_list:
THEMESLIST.append(x) THEMESLIST.append(x)
lower_case_themes_list.append(x.lower()) lower_case_themes_list.append(x.lower())
@ -112,22 +112,22 @@ def get_hugo_themes_list():
def clean_up(): def clean_up():
themes_name_list = get_themes_name_list() themes_name_list = get_themes_name_list()
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
hugo_themes_name_list = [ hugo_themes_name_list = [theme[0] for theme in session.query(Hugothemes.name).all()]
theme[0] for theme in session.query(Hugothemes.name).all()]
for theme_name in hugo_themes_name_list: for theme_name in hugo_themes_name_list:
if theme_name not in themes_name_list: if theme_name not in themes_name_list:
removed_theme = session.query( removed_theme = session.query(Hugothemes).filter_by(name=theme_name).first()
Hugothemes).filter_by(name=theme_name).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
gitlab_themes_name_list = get_gitlab_themes_name_list() gitlab_themes_name_list = get_gitlab_themes_name_list()
hugo_themes_from_gitlab_name_list = [ hugo_themes_from_gitlab_name_list = [
theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()
]
for theme in hugo_themes_from_gitlab_name_list: for theme in hugo_themes_from_gitlab_name_list:
if theme not in gitlab_themes_name_list: if theme not in gitlab_themes_name_list:
removed_theme = session.query( removed_theme = (
Hugothemes_from_gitlab).filter_by(name=theme).first() session.query(Hugothemes_from_gitlab).filter_by(name=theme).first()
)
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
@ -135,29 +135,28 @@ def clean_up():
def dedup_database(): def dedup_database():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
hugo_themes_sha_list = [ hugo_themes_sha_list = [
theme[0] for theme in session.query(Hugothemes.commit_sha).all()] theme[0] for theme in session.query(Hugothemes.commit_sha).all()
]
sha_list = [ sha_list = [
sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count( sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count(sha) > 1
sha
) > 1
] ]
for sha in sha_list: for sha in sha_list:
removed_theme = session.query( removed_theme = session.query(Hugothemes).filter_by(commit_sha=sha).first()
Hugothemes).filter_by(commit_sha=sha).first()
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
hugo_themes_sha_list_from_gitlab = [theme[0] for theme in session.query( hugo_themes_sha_list_from_gitlab = [
Hugothemes_from_gitlab.commit_sha).all()] theme[0] for theme in session.query(Hugothemes_from_gitlab.commit_sha).all()
]
sha_list_from_gitlab = [ sha_list_from_gitlab = [
sha for sha in hugo_themes_sha_list_from_gitlab sha
if hugo_themes_sha_list_from_gitlab.count( for sha in hugo_themes_sha_list_from_gitlab
sha if hugo_themes_sha_list_from_gitlab.count(sha) > 1
) > 1
] ]
for sha in sha_list_from_gitlab: for sha in sha_list_from_gitlab:
removed_theme = session.query( removed_theme = (
Hugothemes_from_gitlab).filter_by(commit_sha=sha).first() session.query(Hugothemes_from_gitlab).filter_by(commit_sha=sha).first()
)
session.delete(removed_theme) session.delete(removed_theme)
session.commit() session.commit()
@ -167,8 +166,9 @@ def parse_gitlab_hugo_themes_list():
gitlab_themes_list = get_gitlab_themes_list() gitlab_themes_list = get_gitlab_themes_list()
for theme in gitlab_themes_list: for theme in gitlab_themes_list:
theme_name = theme[11:] theme_name = theme[11:]
existing_theme = session.query( existing_theme = (
Hugothemes_from_gitlab).filter_by(name=theme_name).first() session.query(Hugothemes_from_gitlab).filter_by(name=theme_name).first()
)
if existing_theme is None: if existing_theme is None:
session.add(Hugothemes_from_gitlab(name=theme_name, url=theme)) session.add(Hugothemes_from_gitlab(name=theme_name, url=theme))
session.commit() session.commit()
@ -179,10 +179,10 @@ def parse_gitlab_hugo_themes_list():
def get_corrected_url(x): def get_corrected_url(x):
if 'neofeed-theme' in x: if "neofeed-theme" in x:
return x.rstrip('/v2') return x.rstrip("/v2")
elif 'docuapi' in x: elif "docuapi" in x:
return x.rstrip('/v2') return x.rstrip("/v2")
else: else:
return x return x
@ -192,8 +192,7 @@ def parse_hugo_themes_list():
for theme in THEMESLIST: for theme in THEMESLIST:
theme_name = theme[11:] theme_name = theme[11:]
theme_url = get_corrected_url(theme) theme_url = get_corrected_url(theme)
existing_theme = session.query( existing_theme = session.query(Hugothemes).filter_by(name=theme_name).first()
Hugothemes).filter_by(name=theme_name).first()
if existing_theme is None: if existing_theme is None:
session.add(Hugothemes(name=theme_name, url=theme_url)) session.add(Hugothemes(name=theme_name, url=theme_url))
session.commit() session.commit()
@ -205,29 +204,27 @@ def parse_hugo_themes_list():
def get_gitlab_project_ids(): def get_gitlab_project_ids():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = (theme[0] for theme in session.query( themes = (theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all())
Hugothemes_from_gitlab.name).all()) match = re.compile(r"(Project ID: )(\d{5,})")
match = re.compile(r'(Project ID: )(\d{5,})')
for theme in themes: for theme in themes:
gitlab_theme = session.query( gitlab_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).one()
Hugothemes_from_gitlab).filter_by(name=theme).one()
if gitlab_theme.gitlab_id is None: if gitlab_theme.gitlab_id is None:
response = get(f"https://{gitlab_theme.url}") response = get(f"https://{gitlab_theme.url}")
if response.status_code == 200: if response.status_code == 200:
gitlab_theme.gitlab_id = match.search(response.text).group(2) gitlab_theme.gitlab_id = match.search(response.text).group(2)
session.commit() session.commit()
if response.status_code == 404: if response.status_code == 404:
print( print(response.status_code, get_gitlab_project_ids.__name__, theme)
response.status_code,
get_gitlab_project_ids.__name__, theme)
print(response.status_code, get_gitlab_project_ids.__name__) print(response.status_code, get_gitlab_project_ids.__name__)
def get_corrected_theme_name(x): def get_corrected_theme_name(x):
if 'neofeed-theme' in x: if "neofeed-theme" in x:
return x.rstrip('/v2') return x.rstrip("/v2")
elif 'docuapi' in x: elif "docuapi" in x:
return x.rstrip('/v2') return x.rstrip("/v2")
elif "osprey-delight" in x:
return x.rstrip("/v5")
else: else:
return x return x
@ -238,14 +235,14 @@ def get_commit_info_for_hugo_themes():
for hugo_theme in theme_names_from_github: for hugo_theme in theme_names_from_github:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
theme_name = get_corrected_theme_name(theme.name) theme_name = get_corrected_theme_name(theme.name)
api_call_url = 'https://api.github.com/repos/' api_call_url = "https://api.github.com/repos/"
api_call_url += f'{theme_name}/commits/{theme.default_branch}' api_call_url += f"{theme_name}/commits/{theme.default_branch}"
if theme.ETag is not None: if theme.ETag is not None:
headers['If-None-Match'] = theme.ETag headers["If-None-Match"] = theme.ETag
else: else:
if 'If-None-Match' in headers: if "If-None-Match" in headers:
del headers['If-None-Match'] del headers["If-None-Match"]
if len(headers) == 0: if len(headers) == 0:
response = get(api_call_url) response = get(api_call_url)
@ -255,21 +252,23 @@ def get_commit_info_for_hugo_themes():
print(hugo_theme) print(hugo_theme)
quit() quit()
if response.status_code == 200: if response.status_code == 200:
theme.ETag = response.headers['ETag'].lstrip('W/') theme.ETag = response.headers["ETag"].lstrip("W/")
result = response.json() result = response.json()
theme.commit_date = result['commit']['author']['date'] theme.commit_date = result["commit"]["author"]["date"]
theme.commit_date_in_seconds = timegm( theme.commit_date_in_seconds = timegm(
strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) strptime(theme.commit_date, "%Y-%m-%dT%H:%M:%SZ")
theme.commit_sha = result['commit']['tree']['sha'] )
theme.commit_sha = result["commit"]["tree"]["sha"]
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print( print(response.status_code, get_commit_info_for_hugo_themes.__name__)
response.status_code, get_commit_info_for_hugo_themes.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print( print(
response.status_code, response.status_code,
get_commit_info_for_hugo_themes.__name__, hugo_theme) get_commit_info_for_hugo_themes.__name__,
hugo_theme,
)
print(response.status_code, get_commit_info_for_hugo_themes.__name__) print(response.status_code, get_commit_info_for_hugo_themes.__name__)
@ -278,36 +277,37 @@ def get_commit_info_for_hugo_themes_from_gitlab():
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
# match = re.compile(r'(\.\d{3})(Z$)') # match = re.compile(r'(\.\d{3})(Z$)')
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
theme = session.query( theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() api_call_url = "https://gitlab.com/api/v4/projects/"
api_call_url = 'https://gitlab.com/api/v4/projects/' api_call_url += f"{theme.gitlab_id}/repository/"
api_call_url += f'{theme.gitlab_id}/repository/' api_call_url += f"commits/{theme.default_branch}"
api_call_url += f'commits/{theme.default_branch}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
# theme.commit_date = (match.sub(r'\2', result[ # theme.commit_date = (match.sub(r'\2', result[
# 'created_at']))[0:19] + 'Z' # 'created_at']))[0:19] + 'Z'
theme.commit_date = result['created_at'][0:19] + 'Z' theme.commit_date = result["created_at"][0:19] + "Z"
theme.commit_date_in_seconds = timegm( theme.commit_date_in_seconds = timegm(
strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) strptime(theme.commit_date, "%Y-%m-%dT%H:%M:%SZ")
theme.commit_sha = result['id'] )
theme.commit_sha = result["id"]
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print( print(
response.status_code, response.status_code,
get_commit_info_for_hugo_themes_from_gitlab.__name__) get_commit_info_for_hugo_themes_from_gitlab.__name__,
)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print( print(
response.status_code, response.status_code,
get_commit_info_for_hugo_themes_from_gitlab.__name__, get_commit_info_for_hugo_themes_from_gitlab.__name__,
hugo_theme hugo_theme,
) )
print( print(
response.status_code, response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__
get_commit_info_for_hugo_themes_from_gitlab.__name__) )
def get_repo_info_for_hugo_themes(): def get_repo_info_for_hugo_themes():
@ -316,31 +316,31 @@ def get_repo_info_for_hugo_themes():
for hugo_theme in theme_names_from_github: for hugo_theme in theme_names_from_github:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
theme_name = get_corrected_theme_name(theme.name) theme_name = get_corrected_theme_name(theme.name)
api_call_url = 'https://api.github.com/repos/' + theme_name api_call_url = "https://api.github.com/repos/" + theme_name
if theme.repo_ETag is not None: if theme.repo_ETag is not None:
headers['If-None-Match'] = theme.repo_ETag headers["If-None-Match"] = theme.repo_ETag
else: else:
if 'If-None-Match' in headers: if "If-None-Match" in headers:
del headers['If-None-Match'] del headers["If-None-Match"]
if len(headers) == 0: if len(headers) == 0:
response = get(api_call_url) response = get(api_call_url)
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
theme.repo_ETag = response.headers['ETag'].lstrip('W/') theme.repo_ETag = response.headers["ETag"].lstrip("W/")
result = response.json() result = response.json()
theme.stargazers_count = result['stargazers_count'] theme.stargazers_count = result["stargazers_count"]
theme.default_branch = result['default_branch'] theme.default_branch = result["default_branch"]
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print(response.status_code, get_repo_info_for_hugo_themes.__name__) print(response.status_code, get_repo_info_for_hugo_themes.__name__)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print( print(
response.status_code, response.status_code, get_repo_info_for_hugo_themes.__name__, hugo_theme
get_repo_info_for_hugo_themes.__name__, hugo_theme) )
print(response.status_code, get_repo_info_for_hugo_themes.__name__) print(response.status_code, get_repo_info_for_hugo_themes.__name__)
@ -348,29 +348,28 @@ def get_repo_info_for_hugo_themes_from_gitlab():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
theme = session.query(Hugothemes_from_gitlab).filter_by( theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
name=hugo_theme).one() api_call_url = f"https://gitlab.com/api/v4/projects/{theme.gitlab_id}"
api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
if theme.star_count != result['star_count']: if theme.star_count != result["star_count"]:
theme.star_count = result['star_count'] theme.star_count = result["star_count"]
if theme.default_branch != result['default_branch']: if theme.default_branch != result["default_branch"]:
theme.default_branch = result['default_branch'] theme.default_branch = result["default_branch"]
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print( print(
response.status_code, response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__
get_repo_info_for_hugo_themes_from_gitlab.__name__) )
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print( print(
response.status_code, response.status_code,
get_repo_info_for_hugo_themes_from_gitlab.__name__, hugo_theme) get_repo_info_for_hugo_themes_from_gitlab.__name__,
print( hugo_theme,
response.status_code, )
get_repo_info_for_hugo_themes_from_gitlab.__name__) print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__)
def get_theme_dot_toml_for_each_hugo_themes(): def get_theme_dot_toml_for_each_hugo_themes():
@ -378,88 +377,94 @@ def get_theme_dot_toml_for_each_hugo_themes():
theme_names_from_github = get_github_themes_name_list() theme_names_from_github = get_github_themes_name_list()
for hugo_theme in theme_names_from_github: for hugo_theme in theme_names_from_github:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
if theme.name == 'gcushen/hugo-academic': if theme.name == "gcushen/hugo-academic":
theme_toml = 'wowchemy/theme.toml' theme_toml = "wowchemy/theme.toml"
else: else:
theme_toml = 'theme.toml' theme_toml = "theme.toml"
theme_name = get_corrected_theme_name(theme.name) theme_name = get_corrected_theme_name(theme.name)
api_call_url = "https://api.github.com/repos/" api_call_url = "https://api.github.com/repos/"
api_call_url += f"{theme_name}/contents/{theme_toml}" api_call_url += f"{theme_name}/contents/{theme_toml}"
if theme.themes_toml_ETag is not None: if theme.themes_toml_ETag is not None:
headers['If-None-Match'] = theme.themes_toml_ETag headers["If-None-Match"] = theme.themes_toml_ETag
else: else:
if 'If-None-Match' in headers: if "If-None-Match" in headers:
del headers['If-None-Match'] del headers["If-None-Match"]
if len(headers) == 0: if len(headers) == 0:
response = get(api_call_url) response = get(api_call_url)
else: else:
response = get(api_call_url, headers=headers) response = get(api_call_url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
theme.themes_toml_ETag = response.headers['ETag'].lstrip('W/') theme.themes_toml_ETag = response.headers["ETag"].lstrip("W/")
result = response.json() result = response.json()
theme.themes_toml_content = result['content'] theme.themes_toml_content = result["content"]
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print( print(
response.status_code, response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__
get_theme_dot_toml_for_each_hugo_themes.__name__) )
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print( print(
response.status_code, response.status_code,
get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme) get_theme_dot_toml_for_each_hugo_themes.__name__,
print( hugo_theme,
response.status_code, )
get_theme_dot_toml_for_each_hugo_themes.__name__) print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__)
def get_theme_dot_toml_for_each_hugo_themes_from_gitlab(): def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
theme = session.query( theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() api_call_url = "https://gitlab.com/api/v4/projects/"
api_call_url = 'https://gitlab.com/api/v4/projects/' api_call_url += f"{theme.gitlab_id}/repository/files/"
api_call_url += f'{theme.gitlab_id}/repository/files/' api_call_url += f"theme.toml?ref={theme.default_branch}"
api_call_url += f'theme.toml?ref={theme.default_branch}'
response = get(api_call_url) response = get(api_call_url)
if response.status_code == 200: if response.status_code == 200:
result = response.json() result = response.json()
theme.themes_toml_content = result['content'] theme.themes_toml_content = result["content"]
session.commit() session.commit()
elif response.status_code == 403: elif response.status_code == 403:
print( print(
response.status_code, response.status_code,
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__,
)
quit() quit()
elif response.status_code == 404: elif response.status_code == 404:
print( print(
response.status_code, response.status_code,
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__,
hugo_theme) hugo_theme,
)
print( print(
response.status_code, response.status_code,
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__,
)
def coalesce_themes(): def coalesce_themes():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
theme_names_from_gitlab = get_gitlab_themes_name_list() theme_names_from_gitlab = get_gitlab_themes_name_list()
for hugo_theme in theme_names_from_gitlab: for hugo_theme in theme_names_from_gitlab:
htfgitlab = session.query( htfgitlab = (
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
)
theme = session.query(Hugothemes).filter_by(name=hugo_theme).first() theme = session.query(Hugothemes).filter_by(name=hugo_theme).first()
if theme is None: if theme is None:
session.add(Hugothemes( session.add(
name=hugo_theme, url=htfgitlab.url, Hugothemes(
commit_sha=htfgitlab.commit_sha, name=hugo_theme,
commit_date=htfgitlab.commit_date, url=htfgitlab.url,
commit_date_in_seconds=htfgitlab.commit_date_in_seconds, commit_sha=htfgitlab.commit_sha,
stargazers_count=htfgitlab.star_count, commit_date=htfgitlab.commit_date,
themes_toml_content=htfgitlab.themes_toml_content, commit_date_in_seconds=htfgitlab.commit_date_in_seconds,
)) stargazers_count=htfgitlab.star_count,
themes_toml_content=htfgitlab.themes_toml_content,
)
)
else: else:
if theme.url != htfgitlab.url: if theme.url != htfgitlab.url:
theme.url = htfgitlab.url theme.url = htfgitlab.url
@ -480,11 +485,11 @@ def get_corrected_tags(tags):
result = [] result = []
correct = True correct = True
for tag in tags: for tag in tags:
if (len(tag) > 50): if len(tag) > 50:
correct = False correct = False
if not correct: if not correct:
for tag in tags: for tag in tags:
result += [x.lstrip() for x in tag.split(',')] result += [x.lstrip() for x in tag.split(",")]
return result return result
else: else:
return tags return tags
@ -493,21 +498,19 @@ def get_corrected_tags(tags):
def parse_themes_toml_for_each_hugo_themes(): def parse_themes_toml_for_each_hugo_themes():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
themes = [theme[0] for theme in session.query(Hugothemes.name).all()] themes = [theme[0] for theme in session.query(Hugothemes.name).all()]
match = re.compile(r'\s(\d+\.\d+\.\d+)\s') match = re.compile(r"\s(\d+\.\d+\.\d+)\s")
for hugo_theme in themes: for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
if theme.themes_toml_content is not None: if theme.themes_toml_content is not None:
content = b64decode(theme.themes_toml_content).decode('utf-8') content = b64decode(theme.themes_toml_content).decode("utf-8")
# put quotes around any unquoted double-dotted version numbers # put quotes around any unquoted double-dotted version numbers
# (and add a newline afterwards) # (and add a newline afterwards)
# because python toml libraries will error out on those # because python toml libraries will error out on those
theme_toml = toml.loads(match.sub(r'"\1"\n', content)) theme_toml = toml.loads(match.sub(r'"\1"\n', content))
if 'tags' in theme_toml: if "tags" in theme_toml:
if len(theme_toml['tags']) > 0: if len(theme_toml["tags"]) > 0:
corrected_tags = get_corrected_tags(theme_toml['tags']) corrected_tags = get_corrected_tags(theme_toml["tags"])
tt = [ tt = [tag.lower() for tag in corrected_tags if len(tag) > 1]
tag.lower() for tag in corrected_tags if len(tag) > 1
]
theme_tags = list(set(tt)) theme_tags = list(set(tt))
if theme.num_tags != len(theme_tags): if theme.num_tags != len(theme_tags):
theme.num_tags = len(theme_tags) theme.num_tags = len(theme_tags)
@ -527,11 +530,12 @@ def parse_themes_toml_for_each_hugo_themes():
theme.tags_list = None theme.tags_list = None
if theme.num_tags != 0: if theme.num_tags != 0:
theme.num_tags = 0 theme.num_tags = 0
if 'features' in theme_toml: if "features" in theme_toml:
if len(theme_toml['features']) > 0: if len(theme_toml["features"]) > 0:
tf = [ tf = [
feature.lower() for feature in feature.lower()
theme_toml['features'] if len(feature) > 1 for feature in theme_toml["features"]
if len(feature) > 1
] ]
theme_features = list(set(tf)) theme_features = list(set(tf))
if theme.num_features != len(theme_features): if theme.num_features != len(theme_features):
@ -552,28 +556,28 @@ def parse_themes_toml_for_each_hugo_themes():
theme.features_list = None theme.features_list = None
if theme.num_features != 0: if theme.num_features != 0:
theme.num_features = 0 theme.num_features = 0
if 'license' in theme_toml: if "license" in theme_toml:
if theme.theme_license != theme_toml['license']: if theme.theme_license != theme_toml["license"]:
theme.theme_license = theme_toml['license'] theme.theme_license = theme_toml["license"]
else: else:
if theme.theme_license is not None: if theme.theme_license is not None:
theme.theme_license = None theme.theme_license = None
if 'min_version' in theme_toml: if "min_version" in theme_toml:
corrected_mv = get_corrected_min_ver(theme_toml['min_version']) corrected_mv = get_corrected_min_ver(theme_toml["min_version"])
if theme.min_ver != corrected_mv: if theme.min_ver != corrected_mv:
theme.min_ver = corrected_mv theme.min_ver = corrected_mv
else: else:
if theme.min_ver is not None: if theme.min_ver is not None:
theme.min_ver = None theme.min_ver = None
if 'description' in theme_toml: if "description" in theme_toml:
if theme.desc != theme_toml['description']: if theme.desc != theme_toml["description"]:
theme.desc = theme_toml['description'] theme.desc = theme_toml["description"]
else: else:
if theme.desc is not None: if theme.desc is not None:
theme.desc = None theme.desc = None
if 'name' in theme_toml: if "name" in theme_toml:
if theme.cname != theme_toml['name']: if theme.cname != theme_toml["name"]:
theme.cname = theme_toml['name'] theme.cname = theme_toml["name"]
else: else:
if theme.cname is not None: if theme.cname is not None:
theme.cname = None theme.cname = None
@ -598,8 +602,8 @@ def parse_themes_toml_for_each_hugo_themes():
def get_corrected_min_ver(x): def get_corrected_min_ver(x):
if 'o' in str(x): if "o" in str(x):
return x.replace('o', '0') return x.replace("o", "0")
else: else:
return x return x
@ -608,41 +612,43 @@ def generate_report():
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
hugo_themes = [ hugo_themes = [
{ {
'name': theme.name, "name": theme.name,
'commit': theme.commit_sha[0:6], "commit": theme.commit_sha[0:6],
'date': theme.commit_date[0:10], "date": theme.commit_date[0:10],
'date_in_seconds': theme.commit_date_in_seconds, "date_in_seconds": theme.commit_date_in_seconds,
'url': f'https://{theme.url}', "url": f"https://{theme.url}",
'short_name': theme.name.split('/')[1], "short_name": theme.name.split("/")[1],
'num_stars': theme.stargazers_count, "num_stars": theme.stargazers_count,
'tags': literal_eval( "tags": literal_eval(theme.tags_list)
theme.tags_list) if theme.tags_list is not None else [], if theme.tags_list is not None
'features': literal_eval( else [],
theme.features_list "features": literal_eval(theme.features_list)
) if theme.features_list is not None else [], if theme.features_list is not None
'license': theme.theme_license if else [],
theme.theme_license is not None else '', "license": theme.theme_license if theme.theme_license is not None else "",
'min_ver': theme.min_ver if theme.min_ver is not None else '', "min_ver": theme.min_ver if theme.min_ver is not None else "",
'desc': theme.desc if theme.desc is not None else '', "desc": theme.desc if theme.desc is not None else "",
'cname': theme.cname if "cname": theme.cname
theme.cname is not None else theme.name.split('/')[1], if theme.cname is not None
} for theme in session.query(Hugothemes).all() else theme.name.split("/")[1],
}
for theme in session.query(Hugothemes).all()
] ]
output = template.render(themes=hugo_themes) output = template.render(themes=hugo_themes)
index_page = open('hugo-themes-report/hugo-themes-report.html', 'w') index_page = open("hugo-themes-report/hugo-themes-report.html", "w")
index_page.write(output) index_page.write(output)
index_page.close() index_page.close()
if __name__ == "__main__": if __name__ == "__main__":
''' """
or test with or test with
` `
python3 -c'import rank_hugo_themes; rank_hugo_themes python3 -c'import rank_hugo_themes; rank_hugo_themes
.parse_themes_toml_for_each_hugo_themes() .parse_themes_toml_for_each_hugo_themes()
; rank_hugo_themes.generate_report()' ; rank_hugo_themes.generate_report()'
` `
''' """
dedup_database() dedup_database()
get_hugo_themes_list() get_hugo_themes_list()
if len(THEMESLIST) > 300: if len(THEMESLIST) > 300: