another major rewrite

This commit is contained in:
Trent Palmer 2021-08-11 11:58:10 -07:00
parent 172800ca1c
commit 562e82bb4e
9 changed files with 513 additions and 5 deletions

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
# rank_hugo_themes.py
from jinja2 import Environment, FileSystemLoader
import re
import toml
from calendar import timegm
@ -17,6 +18,9 @@ from sqlalchemy.orm import deferred, sessionmaker
engine = create_engine('sqlite:///hugothemes.db', echo=False)
Base = declarative_base()
file_loader = FileSystemLoader('templates')
env = Environment(loader=file_loader)
template = env.get_template('base.html')
class Tags(Base):
@ -72,17 +76,19 @@ class Hugothemes(Base):
tags_list = Column(TEXT)
num_tags = Column(Integer)
default_branch = Column(TEXT)
features_list = Column(TEXT)
num_features = Column(Integer)
def __repr__(self):
repr_string = "<(name = '%s', ETag = '%s', url = '%s', commit_sha = '%s', commit_date = '%s'"
repr_string += ", commit_date_in_seconds = '%s', repo_ETag = '%s', stargazers_count = '%s', themes_toml_ETag = '%s'"
repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', default_branch = '%s')>"
repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', default_branch = '%s', features_list = '%s', num_features = '%s')>"
repr_values = (
self.name, self.ETag, self.url,
self.commit_sha, self.commit_date, self.commit_date_in_seconds,
self.repo_ETag, self.stargazers_count, self.themes_toml_ETag,
self.themes_toml_content, self.tags_list,
self.num_tags, self.default_branch
self.num_tags, self.default_branch, self.features_list, self.num_features,
)
return repr_string % repr_values
@ -384,9 +390,51 @@ def coalesce_themes():
session.commit()
def update_features_list_for_each_hugo_themes():
session = sessionmaker(bind=engine)()
themes = [theme[0] for theme in session.query(Hugothemes.name).all()]
match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
if theme.themes_toml_content is not None:
content = b64decode(theme.themes_toml_content).decode('utf-8')
theme_toml = toml.loads(match.sub(r'"\1"\n', content))
if 'features' in theme_toml:
if len(theme_toml['features']) > 0:
theme_features = [feature.lower() for feature in theme_toml['features'] if len(feature) > 0]
if theme.num_features != len(theme_features): theme.num_features = len(theme_features)
if theme.num_features > 0:
if theme.features_list != str(theme_features): theme.features_list = str(theme_features)
else:
if theme.features_list is not None: theme.features_list = None
else:
if theme.features_list is not None: theme.features_list = None
if theme.num_features != 0: theme.num_features = 0
else:
if theme.features_list is not None: theme.features_list = None
if theme.num_features != 0: theme.num_features = 0
else:
if theme.features_list is not None: theme.features_list = None
if theme.num_features != 0: theme.num_features = 0
session.commit()
def get_corrected_tags(tags):
result = []
correct = True
for tag in tags:
if (len(tag) > 50): correct = False
if not correct:
for tag in tags:
result += [x.lstrip() for x in tag.split(',')]
return result
else:
return tags
def update_tags_list_for_each_hugo_themes():
session = sessionmaker(bind=engine)()
themes = [theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name != THEMESLISTREPO).all()]
themes = [theme[0] for theme in session.query(Hugothemes.name).all()]
match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
for hugo_theme in themes:
theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
@ -398,7 +446,8 @@ def update_tags_list_for_each_hugo_themes():
theme_toml = toml.loads(match.sub(r'"\1"\n', content))
if 'tags' in theme_toml:
if len(theme_toml['tags']) > 0:
theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0]
corrected_tags = get_corrected_tags(theme_toml['tags'])
theme_tags = [tag.lower() for tag in corrected_tags if len(tag) > 0]
if theme.num_tags != len(theme_tags): theme.num_tags = len(theme_tags)
if theme.num_tags > 0:
if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags)
@ -519,7 +568,33 @@ def write_reports():
by_date.close()
def generate_report():
session = sessionmaker(bind=engine)()
hugo_themes = [
{
'name': theme.name,
'commit': theme.commit_sha[0:6],
'date': theme.commit_date[0:10],
'date_in_seconds': theme.commit_date_in_seconds,
'url': f'https://{theme.url}',
'short_name': theme.name.split('/')[1],
'num_stars': theme.stargazers_count,
'tags': literal_eval(theme.tags_list) if theme.tags_list is not None else [],
'features': literal_eval(theme.features_list) if theme.features_list is not None else [],
} for theme in session.query(Hugothemes).all()
]
output = template.render(themes=hugo_themes)
index_page = open('hugo-themes-report/hugo-themes-report.html', 'w')
index_page.write(output)
index_page.close()
if __name__ == "__main__":
'''
update_tags_list_for_each_hugo_themes()
update_features_list_for_each_hugo_themes()
generate_report()
'''
get_hugo_themes_list()
if len(THEMESLIST) > 300:
clean_up()
@ -534,5 +609,7 @@ if __name__ == "__main__":
get_theme_dot_toml_for_each_hugo_themes_from_gitlab()
coalesce_themes()
update_tags_list_for_each_hugo_themes()
update_features_list_for_each_hugo_themes()
update_tag_table()
write_reports()
# write_reports()
generate_report()

28
templates/base.html Normal file
View File

@ -0,0 +1,28 @@
<!DOCTYPE html>
<html lang='en'>
<head>
<title>>Hugo Themes Report</title>
<meta charset='utf-8'>
<meta name='viewport' content='width=device-width, initial-scale=1'>
<style>{% include 'css/main.css' %}</style>
</head>
<body>
<h1 id="title">
<a href="https://github.com/TrentSPalmer/hugo_themes_report" target="_blank">Hugo Themes Report</a>
</h1>
<button type="button" id="selection-button" class="collapsible">Select Tags/Features</button>
<div id="selection-menu">
</div>
<div id="results">
</div>
<script>
var themes = {{ themes }};
{% include 'js/selectionMenuCollapse.js' %}
{% include 'js/getAvailableTagsAndFeatures.js' %}
{% include 'js/buildSortByDiv.js' %}
{% include 'js/buildSelectionMenu.js' %}
{% include 'js/buildThemeTableRow.js' %}
{% include 'js/buildPage.js' %}
</script>
</body>
</html>

78
templates/css/main.css Normal file
View File

@ -0,0 +1,78 @@
body {
font-family: sans-serif;
width: 1200px;
max-width: 98%;
margin: 0 auto 0 auto;
}
#selection-menu {
display: none;
}
#title {
text-align: center;
}
.collapsible {
background-color: #eee;
color: #444;
cursor: pointer;
padding: 1.1rem;
width: 100%;
border: none;
text-align: left;
outline: none;
font-size: 1.1rem;
}
.active, .collapsible:hover {
background-color: #ccc;
}
.content {
padding: 0 1.1rem;
display: none;
overflow: hidden;
background-color: #f1f1f1;
}
.collapsible:after {
content: '\02795';
font-size: .8rem;
color: white;
float: right;
margin-left: .4rem;
}
.active:after {
content: '\2796';
}
#resultsTable {
width: 100%;
min-width: 500px;
border-spacing: 0;
padding: 2px;
}
#resultsTable tr{
height: 2rem;
}
#resultsTable tr:nth-child(even) {
background-color: #f2f2f2;
}
#themeTH {
text-align: left;
}
#results {
overflow-x: scroll;
}
td, th {
padding-left: 1rem;
padding-right: 1rem;
text-align: center;
}

84
templates/js/buildPage.js Normal file
View File

@ -0,0 +1,84 @@
function getSortBy() {
let sortByLastCommitInput = document.getElementById('sortByDate');
if (sortByLastCommitInput === null) {
return 'date';
} else {
return sortByLastCommitInput.checked ? 'date' : 'stars';
}
}
function getSortedThemes(themeList, sortedBy) {
if (sortedBy === 'date') {
return themeList.sort((a, b) => b.date_in_seconds - a.date_in_seconds);
} else {
return themeList.sort((a, b) => b.num_stars - a.num_stars);
}
}
function getSelectedTags() {
let tagSelectionInputs = document.getElementsByClassName('tagSelectionInput');
if (tagSelectionInputs.length > 0) {
return [...tagSelectionInputs].filter((x) => x.checked).map((y) => y.value);
} else {
return [];
}
}
function getSelectedFeatures() {
let featureSelectionInputs = document.getElementsByClassName('featureSelectionInput');
if (featureSelectionInputs.length > 0) {
return [...featureSelectionInputs].filter((x) => x.checked).map((y) => y.value);
} else {
return [];
}
}
function getFilteredThemes(selectedTags, selectedFeatures) {
if ((selectedTags.length === 0) && selectedFeatures.length === 0) {
return themes;
} else {
return themes
.filter((x) => selectedTags.every((y) => x.tags.includes(y)) )
.filter((z) => selectedFeatures.every((w) => z.features.includes(w)) );
}
}
function buildResults() {
let resultsDiv = document.getElementById('results');
resultsDiv.innerHTML = '';
let resultsTable = document.createElement("table");
resultsTable.id = 'resultsTable';
resultsTable.style.border = '1px solid black';
resultsTable.style.fontSize = '.9rem';
let resultsTableHeadRow = document.createElement("tr");
resultsDiv.appendChild(resultsTable);
resultsTable.appendChild(resultsTableHeadRow);
let themeTH = document.createElement("th");
themeTH.innerHTML = "theme";
resultsTableHeadRow.appendChild(themeTH);
let dateTH = document.createElement("th");
dateTH.innerHTML = "date";
resultsTableHeadRow.appendChild(dateTH);
let starsTH = document.createElement("th");
starsTH.innerHTML = "stars";
resultsTableHeadRow.appendChild(starsTH);
let commitTH = document.createElement("th");
commitTH.innerHTML = "commit";
resultsTableHeadRow.appendChild(commitTH);
let selectedTags = getSelectedTags();
let selectedFeatures = getSelectedFeatures();
let sortedBy = getSortBy();
let filtered_themes = getFilteredThemes(selectedTags, selectedFeatures);
let sorted_themes = getSortedThemes(filtered_themes, sortedBy);
sorted_themes.forEach(theme => addThemeTableRow(theme));
buildSelectionMenu(sorted_themes, sortedBy, selectedTags, selectedFeatures);
};
buildResults();

View File

@ -0,0 +1,101 @@
function buildTagSelectionInput(tag, selected, tagSelectionRow) {
let tagSelectionInputDiv = document.createElement('div');
tagSelectionInputDiv.style.width = '15rem';
tagSelectionInputDiv.style.maxWidth = '50%';
tagSelectionInputDiv.style.marginTop = '.5rem';
tagSelectionInputDiv.style.marginBottom = '.5rem';
let tagSelectionInput = document.createElement('input');
tagSelectionInput.type = "checkbox";
tagSelectionInput.id = tag.tag + "-selection-input";
tagSelectionInput.name = tag.tag + "-selection-input";
tagSelectionInput.value = tag.tag;
tagSelectionInput.checked = (selected) ? true : false;
tagSelectionInput.classList.add('tagSelectionInput');
tagSelectionInput.onclick = function() { buildResults(); };
tagSelectionInputDiv.appendChild(tagSelectionInput);
let tagSelectionInputLabel = document.createElement('label');
tagSelectionInputLabel.for = tag.tag + "-selection-input";
tagSelectionInputLabel.innerHTML = tag.tag + ' (' + tag.num_themes + ')';
tagSelectionInputDiv.appendChild(tagSelectionInputLabel);
tagSelectionRow.appendChild(tagSelectionInputDiv);
}
function buildTagSelectionDiv(selectedTags, availableTags) {
let selectionMenuDiv = document.getElementById('selection-menu');
let tagSelectionHeading = document.createElement('h2');
tagSelectionHeading.innerHTML = "Select Tags";
selectionMenuDiv.appendChild(tagSelectionHeading);
let tagSelectionRow = document.createElement('div');
tagSelectionRow.style.display = 'flex';
tagSelectionRow.style.flexWrap = 'wrap';
tagSelectionRow.style.justifyContent = 'space-around';
selectionMenuDiv.appendChild(tagSelectionRow);
availableTags
.filter((x) => selectedTags.includes(x.tag))
.forEach((y) => { buildTagSelectionInput(y, true, tagSelectionRow); });
availableTags
.filter((x) => !selectedTags.includes(x.tag))
.forEach((y) => { buildTagSelectionInput(y, false, tagSelectionRow); });
}
function buildFeatureSelectionInput(feature, selected, featureSelectionRow) {
let featureSelectionInputDiv = document.createElement('div');
featureSelectionInputDiv.style.width = '30rem';
featureSelectionInputDiv.style.maxWidth = '50%';
featureSelectionInputDiv.style.marginTop = '.5rem';
featureSelectionInputDiv.style.marginBottom = '.5rem';
let featureSelectionInput = document.createElement('input');
featureSelectionInput.type = "checkbox";
featureSelectionInput.id = feature.feature + "-selection-input";
featureSelectionInput.name = feature.feature + "-selection-input";
featureSelectionInput.value = feature.feature;
featureSelectionInput.checked = (selected) ? true : false;
featureSelectionInput.classList.add('featureSelectionInput');
featureSelectionInput.onclick = function() { buildResults(); };
featureSelectionInputDiv.appendChild(featureSelectionInput);
let featureSelectionInputLabel = document.createElement('label');
featureSelectionInputLabel.for = feature.feature + "-selection-input";
featureSelectionInputLabel.innerHTML = feature.feature + ' (' + feature.num_themes + ')';
featureSelectionInputDiv.appendChild(featureSelectionInputLabel);
featureSelectionRow.appendChild(featureSelectionInputDiv);
}
function buildFeatureSelectionDiv(selectedFeatures, availableFeatures) {
let selectionMenuDiv = document.getElementById('selection-menu');
let featureSelectionHeading = document.createElement('h2');
featureSelectionHeading.innerHTML = "Select Features";
selectionMenuDiv.appendChild(featureSelectionHeading);
let featureSelectionRow = document.createElement('div');
featureSelectionRow.style.display = 'flex';
featureSelectionRow.style.flexWrap = 'wrap';
featureSelectionRow.style.justifyContent = 'space-around';
selectionMenuDiv.appendChild(featureSelectionRow);
availableFeatures
.filter((x) => selectedFeatures.includes(x.feature))
.forEach((y) => { buildFeatureSelectionInput(y, true, featureSelectionRow); });
availableFeatures
.filter((x) => !selectedFeatures.includes(x.feature))
.forEach((y) => { buildFeatureSelectionInput(y, false, featureSelectionRow); });
}
function buildSelectionMenu(sorted_themes, sortedBy, selectedTags, selectedFeatures) {
let availableTags = getAvailableTags(sorted_themes);
let availableFeatures = getAvailableFeatures(sorted_themes);
buildSortByDiv(sortedBy);
buildTagSelectionDiv(selectedTags, availableTags);
buildFeatureSelectionDiv(selectedFeatures, availableFeatures);
}

View File

@ -0,0 +1,53 @@
function buildSortByDiv(sortedBy) {
let menuDiv = document.getElementById('selection-menu');
menuDiv.innerHTML = '';
menuDiv.style.maxWidth = '100%';
let sortByRow = document.createElement('div');
sortByRow.id = 'sortByRow';
sortByRow.style.width = '500px';
sortByRow.style.maxWidth = '100%';
sortByRow.style.display = 'flex';
sortByRow.style.justifyContent = 'space-around';
sortByRow.style.margin = '1rem auto 1rem auto';
let sortByPrompt = document.createElement('div');
sortByPrompt.innerHTML = "Sort By:";
sortByRow.appendChild(sortByPrompt);
let sortByStarsDiv = document.createElement('div');
let sortByStarsInput = document.createElement('input');
sortByStarsInput.type = 'radio';
sortByStarsInput.id = 'sortByStars';
sortByStarsInput.name = 'sortBy';
sortByStarsInput.value = 'stars';
sortByStarsInput.checked = sortedBy === 'stars' ? true : false;
sortByStarsInput.onclick = function() { buildResults(); };
sortByStarsDiv.appendChild(sortByStarsInput);
let sortByStarsLabel = document.createElement('label');
sortByStarsLabel.for = 'stars';
sortByStarsLabel.innerHTML = 'Stars';
sortByStarsDiv.appendChild(sortByStarsLabel);
let sortByLastCommitDiv = document.createElement('div');
let sortByLastCommitInput = document.createElement('input');
sortByLastCommitInput.type = 'radio';
sortByLastCommitInput.id = 'sortByDate';
sortByLastCommitInput.name = 'sortBy';
sortByLastCommitInput.value = 'date';
sortByLastCommitInput.checked = sortedBy === 'date' ? true : false;
sortByLastCommitInput.onclick = function() { buildResults(); };
sortByLastCommitDiv.appendChild(sortByLastCommitInput);
let sortByLastCommitLabel = document.createElement('label');
sortByLastCommitLabel.for = 'date';
sortByLastCommitLabel.innerHTML = 'Latest Commit Date';
sortByLastCommitDiv.appendChild(sortByLastCommitLabel);
sortByRow.appendChild(sortByStarsDiv);
sortByRow.appendChild(sortByLastCommitDiv);
menuDiv.appendChild(sortByRow);
}

View File

@ -0,0 +1,28 @@
function addThemeTableRow(theme) {
let resultsTable = document.getElementById('resultsTable');
let resultsTableRow = document.createElement("tr");
let themeTD = document.createElement("td");
themeTD.innerHTML = '<a href="' + theme.url + '" target="_blank">' + theme.short_name + '</a>';
themeTD.style.whiteSpace = 'nowrap';
themeTD.style.overFlow = 'hidden';
themeTD.style.width = '20%';
resultsTableRow.appendChild(themeTD);
let dateTD = document.createElement("td");
dateTD.innerHTML = theme.date;
dateTD.style.textAlign = 'center';
dateTD.style.minWidth = '8rem';
resultsTableRow.appendChild(dateTD);
let starsTD = document.createElement("td");
starsTD.innerHTML = theme.num_stars;
resultsTableRow.appendChild(starsTD);
let commitTD = document.createElement("td");
commitTD.innerHTML = theme.commit;
commitTD.style.minWidth = '7rem';
resultsTableRow.appendChild(commitTD);
resultsTable.appendChild(resultsTableRow);
};

View File

@ -0,0 +1,48 @@
function getAvailableFeatures(sorted_themes) {
let result = [];
sorted_themes.forEach(x => {
x.features.forEach(feature => {
if (result.length === 0) {
result.push({'feature': feature, 'num_themes': 1});
} else {
let features_in_result = result.map(y => y.feature);
if (features_in_result.includes(feature)) {
result.forEach(w => {
if (w.feature === feature) {
w.num_themes += 1;
}
});
} else {
result.push({'feature': feature, 'num_themes': 1});
}
}
});
});
// return result.sort((a, b) => a.feature.localeCompare(b.feature));
return result.sort((a, b) => b.num_themes - a.num_themes);
}
function getAvailableTags(sorted_themes) {
let result = [];
sorted_themes.forEach(x => {
x.tags.forEach(tag => {
if (result.length === 0) {
result.push({'tag': tag, 'num_themes': 1});
} else {
let tags_in_result = result.map(y => y.tag);
if (tags_in_result.includes(tag)) {
result.forEach(w => {
if (w.tag === tag) {
w.num_themes += 1;
}
});
} else {
result.push({'tag': tag, 'num_themes': 1});
}
}
});
});
return result.sort((a, b) => b.num_themes - a.num_themes);
}

View File

@ -0,0 +1,11 @@
var menuButton = document.getElementById("selection-button");
menuButton.addEventListener("click", function() {
menuButton.classList.toggle("active");
var selectionMenu = document.getElementById("selection-menu");
if (selectionMenu.style.display === "block") {
selectionMenu.style.display = "none";
} else {
selectionMenu.style.display = "block";
}
});