2018-12-19 22:05:08 -08:00
|
|
|
#!/usr/bin/env python3
|
2018-12-06 09:53:55 -08:00
|
|
|
# rank_hugo_themes.py
|
|
|
|
|
2021-08-11 11:58:10 -07:00
|
|
|
from jinja2 import Environment, FileSystemLoader
|
2018-12-19 22:05:08 -08:00
|
|
|
import re
|
2021-08-09 15:39:43 -07:00
|
|
|
import toml
|
2018-12-06 09:53:55 -08:00
|
|
|
from calendar import timegm
|
|
|
|
from time import strptime
|
|
|
|
from requests import get
|
|
|
|
from sys import argv as sys_argv
|
2018-12-19 22:05:08 -08:00
|
|
|
from base64 import b64decode
|
|
|
|
from ast import literal_eval
|
2018-12-06 09:53:55 -08:00
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
from sqlalchemy import create_engine, Column, Integer, VARCHAR, TEXT
|
2019-01-22 02:30:14 -08:00
|
|
|
from sqlalchemy.ext.declarative import declarative_base
|
2021-08-09 15:39:43 -07:00
|
|
|
from sqlalchemy.orm import deferred, sessionmaker
|
2019-01-22 02:30:14 -08:00
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
engine = create_engine('sqlite:///hugothemes.db', echo=False)
|
2019-01-22 02:30:14 -08:00
|
|
|
Base = declarative_base()
|
2021-08-11 11:58:10 -07:00
|
|
|
file_loader = FileSystemLoader('templates')
|
|
|
|
env = Environment(loader=file_loader)
|
|
|
|
template = env.get_template('base.html')
|
2019-01-22 02:30:14 -08:00
|
|
|
|
|
|
|
|
|
|
|
class Hugothemes_from_gitlab(Base):
|
|
|
|
__tablename__ = 'hugothemes_from_gitlab'
|
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
name = Column(VARCHAR, primary_key=True)
|
2019-01-22 02:30:14 -08:00
|
|
|
url = Column(TEXT)
|
|
|
|
commit_sha = Column(TEXT)
|
|
|
|
gitlab_id = Column(TEXT)
|
|
|
|
commit_date_in_seconds = Column(Integer)
|
|
|
|
commit_date = Column(TEXT)
|
|
|
|
star_count = Column(Integer)
|
|
|
|
themes_toml_content = Column(TEXT)
|
2021-08-09 15:39:43 -07:00
|
|
|
default_branch = Column(TEXT)
|
2019-01-22 02:30:14 -08:00
|
|
|
|
|
|
|
def __repr__(self):
|
2021-08-11 20:10:09 -07:00
|
|
|
return f"<Hugothemes_from_gitlab(name={self.name})>"
|
2019-01-22 02:30:14 -08:00
|
|
|
|
|
|
|
|
|
|
|
class Hugothemes(Base):
|
|
|
|
__tablename__ = 'hugothemes'
|
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
name = Column(VARCHAR, primary_key=True)
|
2019-01-22 02:30:14 -08:00
|
|
|
ETag = Column(TEXT)
|
|
|
|
url = Column(TEXT)
|
|
|
|
commit_sha = Column(TEXT)
|
|
|
|
commit_date = Column(TEXT)
|
|
|
|
commit_date_in_seconds = Column(Integer)
|
|
|
|
repo_ETag = Column(TEXT)
|
|
|
|
stargazers_count = Column(Integer)
|
|
|
|
themes_toml_ETag = Column(TEXT)
|
|
|
|
themes_toml_content = deferred(Column(TEXT))
|
|
|
|
tags_list = Column(TEXT)
|
|
|
|
num_tags = Column(Integer)
|
2021-08-09 15:39:43 -07:00
|
|
|
default_branch = Column(TEXT)
|
2021-08-11 11:58:10 -07:00
|
|
|
features_list = Column(TEXT)
|
|
|
|
num_features = Column(Integer)
|
2021-08-13 09:31:43 -07:00
|
|
|
theme_license = Column(TEXT)
|
|
|
|
min_ver = Column(TEXT)
|
|
|
|
desc = Column(TEXT)
|
|
|
|
cname = Column(TEXT)
|
2019-01-22 02:30:14 -08:00
|
|
|
|
|
|
|
def __repr__(self):
|
2021-08-11 20:10:09 -07:00
|
|
|
return f"<Hugothemes(name={self.name})>"
|
2019-01-22 02:30:14 -08:00
|
|
|
|
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
OLDTHEMESLISTREPO = 'gohugoio/hugoThemes'
|
|
|
|
THEMESLISTREPO = 'gohugoio/hugoThemesSiteBuilder'
|
|
|
|
THEMESLIST = []
|
|
|
|
|
|
|
|
|
|
|
|
def get_themes_name_list():
|
|
|
|
return [x[11:] for x in THEMESLIST]
|
|
|
|
|
|
|
|
|
|
|
|
def get_gitlab_themes_list():
|
|
|
|
return [x for x in THEMESLIST if x[0:10] == 'gitlab.com']
|
|
|
|
|
|
|
|
|
|
|
|
def get_gitlab_themes_name_list():
|
|
|
|
return [x[11:] for x in THEMESLIST if x[0:10] == 'gitlab.com']
|
|
|
|
|
|
|
|
|
|
|
|
def get_github_themes_name_list():
|
|
|
|
return [x[11:] for x in THEMESLIST if x[0:10] == 'github.com']
|
2019-01-22 02:30:14 -08:00
|
|
|
|
2018-12-06 09:53:55 -08:00
|
|
|
|
|
|
|
if len(sys_argv) == 2:
|
2021-08-09 15:39:43 -07:00
|
|
|
headers = {'Authorization': 'token ' + sys_argv[1]}
|
2018-12-06 09:53:55 -08:00
|
|
|
else:
|
|
|
|
headers = {}
|
|
|
|
|
|
|
|
|
|
|
|
def get_hugo_themes_list():
|
2021-08-23 20:14:01 -07:00
|
|
|
themes_list_url = "https://raw.githubusercontent.com/"
|
|
|
|
themes_list_url += f"{THEMESLISTREPO}/main/themes.txt"
|
2021-08-09 15:39:43 -07:00
|
|
|
response = get(themes_list_url)
|
2018-12-06 09:53:55 -08:00
|
|
|
|
|
|
|
if response.status_code == 200:
|
2021-08-16 10:34:26 -07:00
|
|
|
lower_case_themes_list = []
|
2021-08-09 15:39:43 -07:00
|
|
|
for x in response.text.splitlines():
|
|
|
|
if (x[0:10] == 'gitlab.com' or x[0:10] == 'github.com'):
|
2021-08-16 10:34:26 -07:00
|
|
|
if x.lower() not in lower_case_themes_list:
|
|
|
|
THEMESLIST.append(x)
|
|
|
|
lower_case_themes_list.append(x.lower())
|
2018-12-24 04:30:18 -08:00
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
print(response.status_code, get_hugo_themes_list.__name__)
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2018-12-06 09:53:55 -08:00
|
|
|
def clean_up():
|
2021-08-09 15:39:43 -07:00
|
|
|
themes_name_list = get_themes_name_list()
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-23 20:14:01 -07:00
|
|
|
hugo_themes_name_list = [
|
|
|
|
theme[0] for theme in session.query(Hugothemes.name).all()]
|
2021-08-09 15:39:43 -07:00
|
|
|
for theme_name in hugo_themes_name_list:
|
|
|
|
if theme_name not in themes_name_list:
|
2021-08-23 20:14:01 -07:00
|
|
|
removed_theme = session.query(
|
|
|
|
Hugothemes).filter_by(name=theme_name).first()
|
2019-01-22 02:30:14 -08:00
|
|
|
session.delete(removed_theme)
|
2021-08-09 15:39:43 -07:00
|
|
|
session.commit()
|
2018-12-24 04:30:18 -08:00
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
gitlab_themes_name_list = get_gitlab_themes_name_list()
|
2021-08-23 20:14:01 -07:00
|
|
|
hugo_themes_from_gitlab_name_list = [
|
|
|
|
theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()]
|
2021-08-09 15:39:43 -07:00
|
|
|
for theme in hugo_themes_from_gitlab_name_list:
|
|
|
|
if theme not in gitlab_themes_name_list:
|
2021-08-23 20:14:01 -07:00
|
|
|
removed_theme = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(name=theme).first()
|
2021-08-09 15:39:43 -07:00
|
|
|
session.delete(removed_theme)
|
|
|
|
session.commit()
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2021-08-16 10:34:26 -07:00
|
|
|
def dedup_database():
|
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-23 20:14:01 -07:00
|
|
|
hugo_themes_sha_list = [
|
|
|
|
theme[0] for theme in session.query(Hugothemes.commit_sha).all()]
|
|
|
|
sha_list = [
|
|
|
|
sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count(
|
|
|
|
sha
|
|
|
|
) > 1
|
|
|
|
]
|
2021-08-16 10:34:26 -07:00
|
|
|
for sha in sha_list:
|
2021-08-23 20:14:01 -07:00
|
|
|
removed_theme = session.query(
|
|
|
|
Hugothemes).filter_by(commit_sha=sha).first()
|
2021-08-16 10:34:26 -07:00
|
|
|
session.delete(removed_theme)
|
|
|
|
session.commit()
|
|
|
|
|
2021-08-23 20:14:01 -07:00
|
|
|
hugo_themes_sha_list_from_gitlab = [theme[0] for theme in session.query(
|
|
|
|
Hugothemes_from_gitlab.commit_sha).all()]
|
|
|
|
sha_list_from_gitlab = [
|
|
|
|
sha for sha in hugo_themes_sha_list_from_gitlab
|
|
|
|
if hugo_themes_sha_list_from_gitlab.count(
|
|
|
|
sha
|
|
|
|
) > 1
|
|
|
|
]
|
2021-08-16 10:34:26 -07:00
|
|
|
for sha in sha_list_from_gitlab:
|
2021-08-23 20:14:01 -07:00
|
|
|
removed_theme = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(commit_sha=sha).first()
|
2021-08-16 10:34:26 -07:00
|
|
|
session.delete(removed_theme)
|
|
|
|
session.commit()
|
|
|
|
|
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
def parse_gitlab_hugo_themes_list():
|
|
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
gitlab_themes_list = get_gitlab_themes_list()
|
|
|
|
for theme in gitlab_themes_list:
|
|
|
|
theme_name = theme[11:]
|
2021-08-23 20:14:01 -07:00
|
|
|
existing_theme = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(name=theme_name).first()
|
2021-08-09 15:39:43 -07:00
|
|
|
if existing_theme is None:
|
|
|
|
session.add(Hugothemes_from_gitlab(name=theme_name, url=theme))
|
2019-01-22 02:30:14 -08:00
|
|
|
session.commit()
|
2021-08-09 15:39:43 -07:00
|
|
|
else:
|
|
|
|
if existing_theme.url != theme:
|
|
|
|
existing_theme.url = theme
|
|
|
|
session.commit()
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2018-12-06 09:53:55 -08:00
|
|
|
def parse_hugo_themes_list():
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-09 15:39:43 -07:00
|
|
|
for theme in THEMESLIST:
|
|
|
|
theme_name = theme[11:]
|
2021-08-23 20:14:01 -07:00
|
|
|
existing_theme = session.query(
|
|
|
|
Hugothemes).filter_by(name=theme_name).first()
|
2021-08-09 15:39:43 -07:00
|
|
|
if existing_theme is None:
|
|
|
|
session.add(Hugothemes(name=theme_name, url=theme))
|
|
|
|
session.commit()
|
|
|
|
else:
|
|
|
|
if existing_theme.url != theme:
|
|
|
|
existing_theme.url = theme
|
2019-01-22 02:30:14 -08:00
|
|
|
session.commit()
|
2018-12-06 09:53:55 -08:00
|
|
|
|
|
|
|
|
2018-12-24 04:30:18 -08:00
|
|
|
def get_gitlab_project_ids():
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-23 20:14:01 -07:00
|
|
|
themes = (theme[0] for theme in session.query(
|
|
|
|
Hugothemes_from_gitlab.name).all())
|
2019-01-22 02:30:14 -08:00
|
|
|
match = re.compile(r'(Project ID: )(\d{5,})')
|
2018-12-24 04:30:18 -08:00
|
|
|
for theme in themes:
|
2021-08-23 20:14:01 -07:00
|
|
|
gitlab_theme = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(name=theme).one()
|
2021-08-09 15:39:43 -07:00
|
|
|
if gitlab_theme.gitlab_id is None:
|
|
|
|
response = get(f"https://{gitlab_theme.url}")
|
|
|
|
if response.status_code == 200:
|
|
|
|
gitlab_theme.gitlab_id = match.search(response.text).group(2)
|
|
|
|
session.commit()
|
|
|
|
if response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_gitlab_project_ids.__name__, theme)
|
2021-08-09 15:39:43 -07:00
|
|
|
print(response.status_code, get_gitlab_project_ids.__name__)
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2018-12-06 09:53:55 -08:00
|
|
|
def get_commit_info_for_hugo_themes():
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-09 15:39:43 -07:00
|
|
|
theme_names_from_github = get_github_themes_name_list()
|
|
|
|
for hugo_theme in theme_names_from_github:
|
2019-01-22 02:30:14 -08:00
|
|
|
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
|
2021-08-23 20:14:01 -07:00
|
|
|
api_call_url = 'https://api.github.com/repos/'
|
|
|
|
api_call_url += f'{theme.name}/commits/{theme.default_branch}'
|
2019-01-22 02:30:14 -08:00
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.ETag is not None:
|
2019-01-22 02:30:14 -08:00
|
|
|
headers['If-None-Match'] = theme.ETag
|
2018-12-06 09:53:55 -08:00
|
|
|
else:
|
|
|
|
if 'If-None-Match' in headers:
|
|
|
|
del headers['If-None-Match']
|
|
|
|
|
|
|
|
if len(headers) == 0:
|
|
|
|
response = get(api_call_url)
|
|
|
|
else:
|
|
|
|
response = get(api_call_url, headers=headers)
|
2021-08-09 15:39:43 -07:00
|
|
|
if response.status_code == 422:
|
|
|
|
print(hugo_theme)
|
|
|
|
quit()
|
2018-12-06 09:53:55 -08:00
|
|
|
if response.status_code == 200:
|
2019-01-22 02:30:14 -08:00
|
|
|
theme.ETag = response.headers['ETag'].lstrip('W/')
|
2018-12-06 09:53:55 -08:00
|
|
|
result = response.json()
|
2019-01-22 02:30:14 -08:00
|
|
|
theme.commit_date = result['commit']['author']['date']
|
2021-08-23 20:14:01 -07:00
|
|
|
theme.commit_date_in_seconds = timegm(
|
|
|
|
strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ'))
|
2021-08-09 15:39:43 -07:00
|
|
|
theme.commit_sha = result['commit']['tree']['sha']
|
2019-01-22 02:30:14 -08:00
|
|
|
session.commit()
|
2018-12-06 09:53:55 -08:00
|
|
|
elif response.status_code == 403:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code, get_commit_info_for_hugo_themes.__name__)
|
2018-12-06 09:53:55 -08:00
|
|
|
quit()
|
2021-08-09 15:39:43 -07:00
|
|
|
elif response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_commit_info_for_hugo_themes.__name__, hugo_theme)
|
2021-08-09 15:39:43 -07:00
|
|
|
print(response.status_code, get_commit_info_for_hugo_themes.__name__)
|
|
|
|
|
2018-12-06 09:53:55 -08:00
|
|
|
|
2018-12-24 04:30:18 -08:00
|
|
|
def get_commit_info_for_hugo_themes_from_gitlab():
|
2021-08-09 15:39:43 -07:00
|
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
theme_names_from_gitlab = get_gitlab_themes_name_list()
|
|
|
|
# match = re.compile(r'(\.\d{3})(Z$)')
|
|
|
|
for hugo_theme in theme_names_from_gitlab:
|
2021-08-23 20:14:01 -07:00
|
|
|
theme = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
|
|
|
|
api_call_url = 'https://gitlab.com/api/v4/projects/'
|
|
|
|
api_call_url += f'{theme.gitlab_id}/repository/'
|
|
|
|
api_call_url += f'commits/{theme.default_branch}'
|
2021-08-09 15:39:43 -07:00
|
|
|
|
|
|
|
response = get(api_call_url)
|
|
|
|
if response.status_code == 200:
|
|
|
|
result = response.json()
|
2021-08-23 20:14:01 -07:00
|
|
|
# theme.commit_date = (match.sub(r'\2', result[
|
|
|
|
# 'created_at']))[0:19] + 'Z'
|
2021-08-09 15:39:43 -07:00
|
|
|
theme.commit_date = result['created_at'][0:19] + 'Z'
|
2021-08-23 20:14:01 -07:00
|
|
|
theme.commit_date_in_seconds = timegm(
|
|
|
|
strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ'))
|
2021-08-09 15:39:43 -07:00
|
|
|
theme.commit_sha = result['id']
|
|
|
|
session.commit()
|
|
|
|
elif response.status_code == 403:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_commit_info_for_hugo_themes_from_gitlab.__name__)
|
2021-08-09 15:39:43 -07:00
|
|
|
quit()
|
|
|
|
elif response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_commit_info_for_hugo_themes_from_gitlab.__name__,
|
|
|
|
hugo_theme
|
|
|
|
)
|
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_commit_info_for_hugo_themes_from_gitlab.__name__)
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2021-08-11 20:10:09 -07:00
|
|
|
def get_repo_info_for_hugo_themes():
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-09 15:39:43 -07:00
|
|
|
theme_names_from_github = get_github_themes_name_list()
|
|
|
|
for hugo_theme in theme_names_from_github:
|
2019-01-22 02:30:14 -08:00
|
|
|
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
|
|
|
|
api_call_url = 'https://api.github.com/repos/' + theme.name
|
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.repo_ETag is not None:
|
2019-01-22 02:30:14 -08:00
|
|
|
headers['If-None-Match'] = theme.repo_ETag
|
2018-12-06 09:53:55 -08:00
|
|
|
else:
|
|
|
|
if 'If-None-Match' in headers:
|
|
|
|
del headers['If-None-Match']
|
|
|
|
|
|
|
|
if len(headers) == 0:
|
|
|
|
response = get(api_call_url)
|
|
|
|
else:
|
|
|
|
response = get(api_call_url, headers=headers)
|
|
|
|
if response.status_code == 200:
|
2019-01-22 02:30:14 -08:00
|
|
|
theme.repo_ETag = response.headers['ETag'].lstrip('W/')
|
2018-12-06 09:53:55 -08:00
|
|
|
result = response.json()
|
2019-01-22 02:30:14 -08:00
|
|
|
theme.stargazers_count = result['stargazers_count']
|
2021-08-09 15:39:43 -07:00
|
|
|
theme.default_branch = result['default_branch']
|
2019-01-22 02:30:14 -08:00
|
|
|
session.commit()
|
2018-12-06 09:53:55 -08:00
|
|
|
elif response.status_code == 403:
|
2021-08-11 20:10:09 -07:00
|
|
|
print(response.status_code, get_repo_info_for_hugo_themes.__name__)
|
2018-12-06 09:53:55 -08:00
|
|
|
quit()
|
2021-08-09 15:39:43 -07:00
|
|
|
elif response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_repo_info_for_hugo_themes.__name__, hugo_theme)
|
2021-08-11 20:10:09 -07:00
|
|
|
print(response.status_code, get_repo_info_for_hugo_themes.__name__)
|
2018-12-06 09:53:55 -08:00
|
|
|
|
2018-12-19 22:05:08 -08:00
|
|
|
|
2021-08-11 20:10:09 -07:00
|
|
|
def get_repo_info_for_hugo_themes_from_gitlab():
|
2021-08-09 15:39:43 -07:00
|
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
theme_names_from_gitlab = get_gitlab_themes_name_list()
|
|
|
|
for hugo_theme in theme_names_from_gitlab:
|
2021-08-23 20:14:01 -07:00
|
|
|
theme = session.query(Hugothemes_from_gitlab).filter_by(
|
|
|
|
name=hugo_theme).one()
|
2021-08-09 15:39:43 -07:00
|
|
|
api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}'
|
|
|
|
response = get(api_call_url)
|
|
|
|
if response.status_code == 200:
|
|
|
|
result = response.json()
|
|
|
|
if theme.star_count != result['star_count']:
|
|
|
|
theme.star_count = result['star_count']
|
|
|
|
if theme.default_branch != result['default_branch']:
|
|
|
|
theme.default_branch = result['default_branch']
|
|
|
|
session.commit()
|
|
|
|
elif response.status_code == 403:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_repo_info_for_hugo_themes_from_gitlab.__name__)
|
2021-08-09 15:39:43 -07:00
|
|
|
quit()
|
|
|
|
elif response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_repo_info_for_hugo_themes_from_gitlab.__name__, hugo_theme)
|
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_repo_info_for_hugo_themes_from_gitlab.__name__)
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2018-12-19 22:05:08 -08:00
|
|
|
def get_theme_dot_toml_for_each_hugo_themes():
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-09 15:39:43 -07:00
|
|
|
theme_names_from_github = get_github_themes_name_list()
|
|
|
|
for hugo_theme in theme_names_from_github:
|
2019-01-22 02:30:14 -08:00
|
|
|
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.name == 'gcushen/hugo-academic':
|
|
|
|
theme_toml = 'wowchemy/theme.toml'
|
|
|
|
else:
|
|
|
|
theme_toml = 'theme.toml'
|
2021-08-23 20:14:01 -07:00
|
|
|
api_call_url = "https://api.github.com/repos/"
|
|
|
|
api_call_url += f"{theme.name}/contents/{theme_toml}"
|
2019-01-22 02:30:14 -08:00
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.themes_toml_ETag is not None:
|
2019-01-22 02:30:14 -08:00
|
|
|
headers['If-None-Match'] = theme.themes_toml_ETag
|
2018-12-19 22:05:08 -08:00
|
|
|
else:
|
|
|
|
if 'If-None-Match' in headers:
|
|
|
|
del headers['If-None-Match']
|
|
|
|
|
|
|
|
if len(headers) == 0:
|
|
|
|
response = get(api_call_url)
|
|
|
|
else:
|
|
|
|
response = get(api_call_url, headers=headers)
|
|
|
|
if response.status_code == 200:
|
2019-01-22 02:30:14 -08:00
|
|
|
theme.themes_toml_ETag = response.headers['ETag'].lstrip('W/')
|
2018-12-19 22:05:08 -08:00
|
|
|
result = response.json()
|
2019-01-22 02:30:14 -08:00
|
|
|
theme.themes_toml_content = result['content']
|
|
|
|
session.commit()
|
2018-12-19 22:05:08 -08:00
|
|
|
elif response.status_code == 403:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes.__name__)
|
2018-12-19 22:05:08 -08:00
|
|
|
quit()
|
2021-08-09 15:39:43 -07:00
|
|
|
elif response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme)
|
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes.__name__)
|
2018-12-06 09:53:55 -08:00
|
|
|
|
2018-12-19 22:05:08 -08:00
|
|
|
|
2018-12-24 04:30:18 -08:00
|
|
|
def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
|
2021-08-09 15:39:43 -07:00
|
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
theme_names_from_gitlab = get_gitlab_themes_name_list()
|
|
|
|
for hugo_theme in theme_names_from_gitlab:
|
2021-08-23 20:14:01 -07:00
|
|
|
theme = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
|
|
|
|
api_call_url = 'https://gitlab.com/api/v4/projects/'
|
|
|
|
api_call_url += f'{theme.gitlab_id}/repository/files/'
|
|
|
|
api_call_url += f'theme.toml?ref={theme.default_branch}'
|
2021-08-09 15:39:43 -07:00
|
|
|
response = get(api_call_url)
|
|
|
|
if response.status_code == 200:
|
|
|
|
result = response.json()
|
|
|
|
theme.themes_toml_content = result['content']
|
|
|
|
session.commit()
|
|
|
|
elif response.status_code == 403:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
|
2021-08-09 15:39:43 -07:00
|
|
|
quit()
|
|
|
|
elif response.status_code == 404:
|
2021-08-23 20:14:01 -07:00
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__,
|
|
|
|
hugo_theme)
|
|
|
|
print(
|
|
|
|
response.status_code,
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__)
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
|
|
|
def coalesce_themes():
|
2021-08-09 15:39:43 -07:00
|
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
theme_names_from_gitlab = get_gitlab_themes_name_list()
|
|
|
|
for hugo_theme in theme_names_from_gitlab:
|
2021-08-23 20:14:01 -07:00
|
|
|
htfgitlab = session.query(
|
|
|
|
Hugothemes_from_gitlab).filter_by(name=hugo_theme).one()
|
2021-08-09 15:39:43 -07:00
|
|
|
theme = session.query(Hugothemes).filter_by(name=hugo_theme).first()
|
|
|
|
if theme is None:
|
|
|
|
session.add(Hugothemes(
|
2021-08-23 20:14:01 -07:00
|
|
|
name=hugo_theme, url=htfgitlab.url,
|
|
|
|
commit_sha=htfgitlab.commit_sha,
|
|
|
|
commit_date=htfgitlab.commit_date,
|
|
|
|
commit_date_in_seconds=htfgitlab.commit_date_in_seconds,
|
|
|
|
stargazers_count=htfgitlab.star_count,
|
|
|
|
themes_toml_content=htfgitlab.themes_toml_content,
|
2021-08-09 15:39:43 -07:00
|
|
|
))
|
|
|
|
else:
|
|
|
|
if theme.url != htfgitlab.url: theme.url = htfgitlab.url
|
2021-08-23 20:14:01 -07:00
|
|
|
if theme.commit_sha != htfgitlab.commit_sha:
|
|
|
|
theme.commit_sha = htfgitlab.commit_sha
|
|
|
|
if theme.commit_date != htfgitlab.commit_date:
|
|
|
|
theme.commit_date = htfgitlab.commit_date
|
|
|
|
htfgitlab_cdis = htfgitlab.commit_date_in_seconds
|
|
|
|
if theme.commit_date_in_seconds != htfgitlab_cdis:
|
|
|
|
theme.commit_date_in_seconds = htfgitlab_cdis
|
|
|
|
if theme.stargazers_count != htfgitlab.star_count:
|
|
|
|
theme.stargazers_count = htfgitlab.star_count
|
2021-08-09 15:39:43 -07:00
|
|
|
theme.themes_toml_content = htfgitlab.themes_toml_content
|
|
|
|
session.commit()
|
2018-12-24 04:30:18 -08:00
|
|
|
|
|
|
|
|
2021-08-11 11:58:10 -07:00
|
|
|
def get_corrected_tags(tags):
|
|
|
|
result = []
|
|
|
|
correct = True
|
|
|
|
for tag in tags:
|
|
|
|
if (len(tag) > 50): correct = False
|
|
|
|
if not correct:
|
|
|
|
for tag in tags:
|
|
|
|
result += [x.lstrip() for x in tag.split(',')]
|
|
|
|
return result
|
|
|
|
else:
|
|
|
|
return tags
|
|
|
|
|
|
|
|
|
2021-08-13 09:31:43 -07:00
|
|
|
def parse_themes_toml_for_each_hugo_themes():
|
2019-01-22 02:30:14 -08:00
|
|
|
session = sessionmaker(bind=engine)()
|
2021-08-11 11:58:10 -07:00
|
|
|
themes = [theme[0] for theme in session.query(Hugothemes.name).all()]
|
2019-01-22 02:30:14 -08:00
|
|
|
match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
|
|
|
|
for hugo_theme in themes:
|
|
|
|
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.themes_toml_content is not None:
|
2019-01-22 02:30:14 -08:00
|
|
|
content = b64decode(theme.themes_toml_content).decode('utf-8')
|
2018-12-19 22:05:08 -08:00
|
|
|
# put quotes around any unquoted double-dotted version numbers
|
2020-02-14 15:42:20 -08:00
|
|
|
# (and add a newline afterwards)
|
2018-12-19 22:05:08 -08:00
|
|
|
# because python toml libraries will error out on those
|
2021-08-09 15:39:43 -07:00
|
|
|
theme_toml = toml.loads(match.sub(r'"\1"\n', content))
|
2018-12-19 22:05:08 -08:00
|
|
|
if 'tags' in theme_toml:
|
|
|
|
if len(theme_toml['tags']) > 0:
|
2021-08-11 11:58:10 -07:00
|
|
|
corrected_tags = get_corrected_tags(theme_toml['tags'])
|
2021-08-23 20:14:01 -07:00
|
|
|
theme_tags = [
|
|
|
|
tag.lower() for tag in corrected_tags if len(tag) > 1]
|
|
|
|
if theme.num_tags != len(theme_tags):
|
|
|
|
theme.num_tags = len(theme_tags)
|
2019-01-22 02:30:14 -08:00
|
|
|
if theme.num_tags > 0:
|
2021-08-23 20:14:01 -07:00
|
|
|
if theme.tags_list != str(theme_tags):
|
|
|
|
theme.tags_list = str(theme_tags)
|
2018-12-19 22:05:08 -08:00
|
|
|
else:
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.tags_list is not None: theme.tags_list = None
|
2018-12-19 22:05:08 -08:00
|
|
|
else:
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.tags_list is not None: theme.tags_list = None
|
2019-01-22 02:30:14 -08:00
|
|
|
if theme.num_tags != 0: theme.num_tags = 0
|
2018-12-19 22:05:08 -08:00
|
|
|
else:
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.tags_list is not None: theme.tags_list = None
|
2019-01-22 02:30:14 -08:00
|
|
|
if theme.num_tags != 0: theme.num_tags = 0
|
2021-08-13 09:31:43 -07:00
|
|
|
if 'features' in theme_toml:
|
|
|
|
if len(theme_toml['features']) > 0:
|
2021-08-23 20:14:01 -07:00
|
|
|
theme_features = [
|
|
|
|
feature.lower() for feature in theme_toml[
|
|
|
|
'features'] if len(feature) > 1]
|
|
|
|
if theme.num_features != len(theme_features):
|
|
|
|
theme.num_features = len(theme_features)
|
2021-08-13 09:31:43 -07:00
|
|
|
if theme.num_features > 0:
|
2021-08-23 20:14:01 -07:00
|
|
|
if theme.features_list != str(theme_features):
|
|
|
|
theme.features_list = str(theme_features)
|
2021-08-13 09:31:43 -07:00
|
|
|
else:
|
2021-08-23 20:14:01 -07:00
|
|
|
if theme.features_list is not None:
|
|
|
|
theme.features_list = None
|
2021-08-13 09:31:43 -07:00
|
|
|
else:
|
2021-08-23 20:14:01 -07:00
|
|
|
if theme.features_list is not None:
|
|
|
|
theme.features_list = None
|
2021-08-13 09:31:43 -07:00
|
|
|
if theme.num_features != 0: theme.num_features = 0
|
|
|
|
else:
|
|
|
|
if theme.features_list is not None: theme.features_list = None
|
|
|
|
if theme.num_features != 0: theme.num_features = 0
|
|
|
|
if 'license' in theme_toml:
|
|
|
|
if theme.theme_license != theme_toml['license']:
|
|
|
|
theme.theme_license = theme_toml['license']
|
|
|
|
else:
|
|
|
|
if theme.theme_license is not None: theme.theme_license = None
|
|
|
|
if 'min_version' in theme_toml:
|
|
|
|
if theme.min_ver != theme_toml['min_version']:
|
|
|
|
theme.min_ver = theme_toml['min_version']
|
|
|
|
else:
|
|
|
|
if theme.min_ver is not None: theme.min_ver = None
|
|
|
|
if 'description' in theme_toml:
|
|
|
|
if theme.desc != theme_toml['description']:
|
|
|
|
theme.desc = theme_toml['description']
|
|
|
|
else:
|
|
|
|
if theme.desc is not None: theme.desc = None
|
|
|
|
if 'name' in theme_toml:
|
|
|
|
if theme.cname != theme_toml['name']:
|
|
|
|
theme.cname = theme_toml['name']
|
|
|
|
else:
|
|
|
|
if theme.cname is not None: theme.cname = None
|
2018-12-19 22:05:08 -08:00
|
|
|
else:
|
2021-08-09 15:39:43 -07:00
|
|
|
if theme.tags_list is not None: theme.tags_list = None
|
2019-01-22 02:30:14 -08:00
|
|
|
if theme.num_tags != 0: theme.num_tags = 0
|
2021-08-13 09:31:43 -07:00
|
|
|
if theme.features_list is not None: theme.features_list = None
|
|
|
|
if theme.num_features != 0: theme.num_features = 0
|
|
|
|
if theme.license is not None: theme.license = None
|
|
|
|
if theme.min_ver is not None: theme.min_ver = None
|
|
|
|
if theme.desc is not None: theme.desc = None
|
|
|
|
if theme.cname is not None: theme.cname = None
|
2019-01-22 02:30:14 -08:00
|
|
|
session.commit()
|
2018-12-19 22:05:08 -08:00
|
|
|
|
|
|
|
|
2021-08-11 11:58:10 -07:00
|
|
|
def generate_report():
|
|
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
hugo_themes = [
|
|
|
|
{
|
|
|
|
'name': theme.name,
|
|
|
|
'commit': theme.commit_sha[0:6],
|
|
|
|
'date': theme.commit_date[0:10],
|
|
|
|
'date_in_seconds': theme.commit_date_in_seconds,
|
|
|
|
'url': f'https://{theme.url}',
|
|
|
|
'short_name': theme.name.split('/')[1],
|
|
|
|
'num_stars': theme.stargazers_count,
|
2021-08-23 20:14:01 -07:00
|
|
|
'tags': literal_eval(
|
|
|
|
theme.tags_list) if theme.tags_list is not None else [],
|
|
|
|
'features': literal_eval(
|
|
|
|
theme.features_list
|
|
|
|
) if theme.features_list is not None else [],
|
|
|
|
'license': theme.theme_license if
|
|
|
|
theme.theme_license is not None else '',
|
2021-08-13 09:31:43 -07:00
|
|
|
'min_ver': theme.min_ver if theme.min_ver is not None else '',
|
|
|
|
'desc': theme.desc if theme.desc is not None else '',
|
2021-08-23 20:14:01 -07:00
|
|
|
'cname': theme.cname if
|
|
|
|
theme.cname is not None else theme.name.split('/')[1],
|
2021-08-11 11:58:10 -07:00
|
|
|
} for theme in session.query(Hugothemes).all()
|
|
|
|
]
|
|
|
|
output = template.render(themes=hugo_themes)
|
|
|
|
index_page = open('hugo-themes-report/hugo-themes-report.html', 'w')
|
|
|
|
index_page.write(output)
|
|
|
|
index_page.close()
|
|
|
|
|
|
|
|
|
2021-08-09 15:39:43 -07:00
|
|
|
if __name__ == "__main__":
|
2021-08-11 11:58:10 -07:00
|
|
|
'''
|
2021-08-14 08:01:41 -07:00
|
|
|
or test with
|
|
|
|
`
|
2021-08-23 20:14:01 -07:00
|
|
|
python3 -c'import rank_hugo_themes; rank_hugo_themes
|
|
|
|
.parse_themes_toml_for_each_hugo_themes()
|
|
|
|
; rank_hugo_themes.generate_report()'
|
2021-08-14 08:01:41 -07:00
|
|
|
`
|
2021-08-11 11:58:10 -07:00
|
|
|
'''
|
2021-08-16 10:34:26 -07:00
|
|
|
dedup_database()
|
2018-12-06 09:53:55 -08:00
|
|
|
get_hugo_themes_list()
|
2021-08-09 15:39:43 -07:00
|
|
|
if len(THEMESLIST) > 300:
|
|
|
|
clean_up()
|
|
|
|
parse_hugo_themes_list()
|
|
|
|
parse_gitlab_hugo_themes_list()
|
|
|
|
get_gitlab_project_ids()
|
2021-08-11 20:10:09 -07:00
|
|
|
get_repo_info_for_hugo_themes()
|
|
|
|
get_repo_info_for_hugo_themes_from_gitlab()
|
2021-08-09 15:39:43 -07:00
|
|
|
get_commit_info_for_hugo_themes()
|
|
|
|
get_commit_info_for_hugo_themes_from_gitlab()
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes()
|
|
|
|
get_theme_dot_toml_for_each_hugo_themes_from_gitlab()
|
|
|
|
coalesce_themes()
|
2021-08-13 09:31:43 -07:00
|
|
|
parse_themes_toml_for_each_hugo_themes()
|
2021-08-11 11:58:10 -07:00
|
|
|
generate_report()
|