Initial commit

This commit is contained in:
Sijmen 2018-10-19 09:25:54 +02:00
commit 8198de3b50
24 changed files with 8170 additions and 0 deletions

226
.gitignore vendored Normal file
View File

@ -0,0 +1,226 @@
# Created by https://www.gitignore.io/api/vim,macos,emacs,python
### Emacs ###
# -*- mode: gitignore; -*-
*~
\#*\#
/.emacs.desktop
/.emacs.desktop.lock
*.elc
auto-save-list
tramp
.\#*
# Org-mode
.org-id-locations
*_archive
# flymake-mode
*_flymake.*
# eshell files
/eshell/history
/eshell/lastdir
# elpa packages
/elpa/
# reftex files
*.rel
# AUCTeX auto folder
/auto/
# cask packages
.cask/
dist/
# Flycheck
flycheck_*.el
# server auth directory
/server/
# projectiles files
.projectile
# directory configuration
.dir-locals.el
### macOS ###
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
### Python Patch ###
.venv/
### Python.VirtualEnv Stack ###
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
pip-selfcheck.json
### Vim ###
# Swap
[._]*.s[a-v][a-z]
[._]*.sw[a-p]
[._]s[a-rt-v][a-z]
[._]ss[a-gi-z]
[._]sw[a-p]
# Session
Session.vim
# Temporary
.netrwhist
# Auto-generated tag files
tags
# Persistent undo
[._]*.un~
# End of https://www.gitignore.io/api/vim,macos,emacs,python
data/

123
backend/app.py Normal file
View File

@ -0,0 +1,123 @@
from datetime import datetime
from flask import Flask, jsonify, request
from elasticsearch_dsl import Document, Date, Integer, Keyword, Text, FacetedSearch, TermsFacet, DateHistogramFacet
from elasticsearch_dsl.connections import connections
from tqdm import tqdm
import csv
import click
import re
class Question(Document):
title = Text(analyzer="snowball")
body = Text(analyzer="snowball")
category = Keyword()
date = Date()
class Index:
name = "goeievraag"
def save(self, **kwargs):
return super(Question, self).save(**kwargs)
@property
def url(self):
id_ = self.meta.id
if not self.category:
return f"https://www.startpagina.nl/v/vraag/{id_}"
category = self.category.lower()
category = re.sub("[^A-Za-z ]", "", category).replace(" ", "-")
return f"https://www.startpagina.nl/v/{category}/vraag/{id_}"
@property
def summary(self, length=128):
if len(self.body) > length:
return self.body[:length - 3] + "..."
return self.body
class QuestionSearch(FacetedSearch):
doc_types = Question,
fields = "category", "title", "body"
facets = {
"date_frequency": DateHistogramFacet(field="date", interval="month"),
"category": TermsFacet(field="category")
}
app = Flask(__name__)
connections.create_connection(hosts=["localhost"])
Question.init()
def round_sigfig(value, figures):
return float(format(value, f".{figures}g"))
@app.route("/api/")
def index():
query = request.args.get("q")
categories = request.args.get("categories", None)
facets = {}
if categories is not None:
category_list = categories.split(",")
facets["category"] = category_list
search = QuestionSearch(query, facets)
response = search.execute()
date_facets = [{"timestamp": date.timestamp(), "count": count}
for date, count, _ in response.facets.date_frequency]
category_facets = [{"category": category, "count": round_sigfig(count, 3)}
for category, count, _ in response.facets.category]
results = [{"id": hit.meta.id, "score": hit.meta.score, "title": hit.title,
"body": hit.summary, "category": hit.category,
"date": hit.date, "url": hit.url}
for hit in response]
return jsonify(
facets={"months": date_facets, "categories": category_facets},
results=results,
hits=round_sigfig(response.hits.total, 4),
took=response.took / 1000,
)
@app.cli.command()
@click.argument("questions")
@click.argument("categories")
def import_data(questions, categories):
categories_dict = {}
num_lines = sum(1 for line in open(categories))
with open(categories, newline="") as csv_file:
reader = csv.reader(csv_file)
for row in tqdm(reader, desc="Reading categories", total=num_lines):
id_ = int(row[0])
category = row[2]
categories_dict[id_] = category
num_lines = sum(1 for line in open(questions))
with open(questions, newline="") as csv_file:
reader = csv.reader(csv_file)
it = tqdm(reader, desc="Reading questions", total=num_lines)
for i, row in enumerate(it):
try:
id_ = int(row[0])
category_id = int(row[3])
question = Question(meta={"id": id_})
question.date = row[1]
question.category = categories_dict[category_id]
question.title = row[4]
question.body = "\n".join(row[5:])
question.save()
except (IndexError, ValueError):
continue

4
backend/requirements.txt Normal file
View File

@ -0,0 +1,4 @@
elasticsearch-dsl
flask
tqdm
beeprint

7
docker-compose.yml Normal file
View File

@ -0,0 +1,7 @@
version: "3"
services:
elasticsearch:
image: elasticsearch:6.4.2
ports:
- 127.0.0.1:9200:9200
- 127.0.0.1:9300:9300

12
frontend/.babelrc Normal file
View File

@ -0,0 +1,12 @@
{
"presets": [
["env", {
"modules": false,
"targets": {
"browsers": ["> 1%", "last 2 versions", "not ie <= 8"]
}
}],
"stage-2"
],
"plugins": ["transform-vue-jsx", "transform-runtime", "transform-decorators-legacy"]
}

9
frontend/.editorconfig Normal file
View File

@ -0,0 +1,9 @@
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true

4
frontend/.eslintignore Normal file
View File

@ -0,0 +1,4 @@
/build/
/config/
/dist/
/*.js

34
frontend/.eslintrc.js Normal file
View File

@ -0,0 +1,34 @@
// https://eslint.org/docs/user-guide/configuring
module.exports = {
root: true,
parserOptions: {
parser: "babel-eslint"
},
env: {
browser: true,
},
extends: [
// https://github.com/vuejs/eslint-plugin-vue#priority-a-essential-error-prevention
// consider switching to `plugin:vue/strongly-recommended` or `plugin:vue/recommended` for stricter rules.
"plugin:vue/essential",
// https://github.com/standard/standard/blob/master/docs/RULES-en.md
"standard"
],
// required to lint *.vue files
plugins: [
"vue"
],
// add your custom rules here
rules: {
// allow async-await
"generator-star-spacing": "off",
// allow debugger during development
"no-debugger": process.env.NODE_ENV === "production" ? "error" : "off",
"quotes": ["warn", "double"],
"semi": ["warn", "always"],
"comma-dangle": ["warn", "always-multiline"],
"space-before-function-paren": ["warn", "never"],
}
}

14
frontend/.gitignore vendored Normal file
View File

@ -0,0 +1,14 @@
.DS_Store
node_modules/
/dist/
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Editor directories and files
.idea
.vscode
*.suo
*.ntvs*
*.njsproj
*.sln

10
frontend/.postcssrc.js Normal file
View File

@ -0,0 +1,10 @@
// https://github.com/michael-ciniawsky/postcss-load-config
module.exports = {
"plugins": {
"postcss-import": {},
"postcss-url": {},
// to edit target browsers: use "browserslist" field in package.json
"autoprefixer": {}
}
}

21
frontend/README.md Normal file
View File

@ -0,0 +1,21 @@
# zoekmachine
> Een zoekmachine
## Build Setup
``` bash
# install dependencies
npm install
# serve with hot reload at localhost:8080
npm run dev
# build for production with minification
npm run build
# build for production and view the bundle analyzer report
npm run build --report
```
For a detailed explanation on how things work, check out the [guide](http://vuejs-templates.github.io/webpack/) and [docs for vue-loader](http://vuejs.github.io/vue-loader).

View File

@ -0,0 +1,7 @@
"use strict";
const merge = require("webpack-merge");
const prodEnv = require("./prod.env");
module.exports = merge(prodEnv, {
NODE_ENV: '"development"'
});

78
frontend/config/index.js Normal file
View File

@ -0,0 +1,78 @@
'use strict'
// Template version: 1.3.1
// see http://vuejs-templates.github.io/webpack for documentation.
const path = require('path')
module.exports = {
dev: {
// Paths
assetsSubDirectory: 'static',
assetsPublicPath: '/',
// Various Dev Server settings
host: 'localhost', // can be overwritten by process.env.HOST
port: 8080, // can be overwritten by process.env.PORT, if port is in use, a free one will be determined
autoOpenBrowser: false,
errorOverlay: true,
notifyOnErrors: true,
poll: false, // https://webpack.js.org/configuration/dev-server/#devserver-watchoptions-
proxyTable: {
"/api": "http://localhost:5000",
},
// Use Eslint Loader?
// If true, your code will be linted during bundling and
// linting errors and warnings will be shown in the console.
useEslint: true,
// If true, eslint errors and warnings will also be shown in the error overlay
// in the browser.
showEslintErrorsInOverlay: false,
/**
* Source Maps
*/
// https://webpack.js.org/configuration/devtool/#development
devtool: 'cheap-module-eval-source-map',
// If you have problems debugging vue-files in devtools,
// set this to false - it *may* help
// https://vue-loader.vuejs.org/en/options.html#cachebusting
cacheBusting: true,
cssSourceMap: true
},
build: {
// Template for index.html
index: path.resolve(__dirname, '../dist/index.html'),
// Paths
assetsRoot: path.resolve(__dirname, '../dist'),
assetsSubDirectory: 'static',
assetsPublicPath: '/',
/**
* Source Maps
*/
productionSourceMap: true,
// https://webpack.js.org/configuration/devtool/#production
devtool: '#source-map',
// Gzip off by default as many popular static hosts such as
// Surge or Netlify already gzip all static assets for you.
// Before setting to `true`, make sure to:
// npm install --save-dev compression-webpack-plugin
productionGzip: false,
productionGzipExtensions: ['js', 'css'],
// Run the build command with an extra argument to
// View the bundle analyzer report after build finishes:
// `npm run build --report`
// Set to `true` or `false` to always turn it on or off
bundleAnalyzerReport: process.env.npm_config_report
}
}

View File

@ -0,0 +1,4 @@
'use strict'
module.exports = {
NODE_ENV: '"production"'
}

12
frontend/index.html Normal file
View File

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1.0">
<title>zoekmachine</title>
</head>
<body>
<div id="app"></div>
<!-- built files will be auto injected -->
</body>
</html>

76
frontend/package.json Normal file
View File

@ -0,0 +1,76 @@
{
"name": "zoekmachine",
"version": "1.0.0",
"description": "Een zoekmachine",
"author": "Sijmen Schoon <me@sijmenschoon.nl>",
"private": true,
"scripts": {
"dev": "webpack-dev-server --inline --progress --config build/webpack.dev.conf.js",
"start": "npm run dev",
"lint": "eslint --ext .js,.vue src",
"build": "node build/build.js"
},
"dependencies": {
"babel-plugin-transform-decorators-legacy": "^1.3.5",
"vue": "^2.5.2",
"vue-class-component": "^6.3.2",
"vue-property-decorator": "^7.2.0",
"vue-router": "^3.0.1"
},
"devDependencies": {
"autoprefixer": "^7.1.2",
"babel-core": "^6.22.1",
"babel-eslint": "^8.2.1",
"babel-helper-vue-jsx-merge-props": "^2.0.3",
"babel-loader": "^7.1.1",
"babel-plugin-syntax-jsx": "^6.18.0",
"babel-plugin-transform-runtime": "^6.22.0",
"babel-plugin-transform-vue-jsx": "^3.5.0",
"babel-preset-env": "^1.3.2",
"babel-preset-stage-2": "^6.22.0",
"chalk": "^2.0.1",
"copy-webpack-plugin": "^4.0.1",
"css-loader": "^0.28.0",
"eslint": "^4.15.0",
"eslint-config-standard": "^10.2.1",
"eslint-friendly-formatter": "^3.0.0",
"eslint-loader": "^1.7.1",
"eslint-plugin-import": "^2.7.0",
"eslint-plugin-node": "^5.2.0",
"eslint-plugin-promise": "^3.4.0",
"eslint-plugin-standard": "^3.0.1",
"eslint-plugin-vue": "^4.0.0",
"extract-text-webpack-plugin": "^3.0.0",
"file-loader": "^1.1.4",
"friendly-errors-webpack-plugin": "^1.6.1",
"html-webpack-plugin": "^2.30.1",
"node-notifier": "^5.1.2",
"optimize-css-assets-webpack-plugin": "^3.2.0",
"ora": "^1.2.0",
"portfinder": "^1.0.13",
"postcss-import": "^11.0.0",
"postcss-loader": "^2.0.8",
"postcss-url": "^7.2.1",
"rimraf": "^2.6.0",
"semver": "^5.3.0",
"shelljs": "^0.7.6",
"uglifyjs-webpack-plugin": "^1.1.1",
"url-loader": "^0.5.8",
"vue-loader": "^13.3.0",
"vue-style-loader": "^3.0.1",
"vue-template-compiler": "^2.5.2",
"webpack": "^3.6.0",
"webpack-bundle-analyzer": "^2.9.0",
"webpack-dev-server": "^2.9.1",
"webpack-merge": "^4.1.0"
},
"engines": {
"node": ">= 6.0.0",
"npm": ">= 3.0.0"
},
"browserslist": [
"> 1%",
"last 2 versions",
"not ie <= 8"
]
}

38
frontend/src/App.vue Normal file
View File

@ -0,0 +1,38 @@
<template>
<div id="app">
<router-view/>
</div>
</template>
<style>
* {
box-sizing: border-box;
}
body {
margin: 0;
font-size: 13px;
}
#app {
font-family: Arial, sans-serif;
color: #2c3e50;
}
.query__input:hover,
.query__input:focus {
box-shadow: 0 3px 8px 0 rgba(0,0,0,0.2),0 0 0 1px rgba(0,0,0,0.08);
}
.query__input {
width: 500px;
height: 34px;
line-height: 34px;
font-size: 16px;
border: 0;
border-radius: 2px;
box-shadow: 0 2px 2px 0 rgba(0,0,0,0.16),0 0 0 1px rgba(0,0,0,0.08);
transition: box-shadow 200ms cubic-bezier(0.4, 0.0, 0.2, 1);
padding: 5px 9px 5px 16px;
}
</style>

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

View File

@ -0,0 +1,50 @@
<template>
<div class="index">
<div class="logo">
<img class="logo__image" src="@/assets/logo.png">
</div>
<div class="query">
<input class="query__input">
<div class="query__buttons">
<input type="button" value="Goeievraagle zoeken" class="query__submit">
<input type="button" value="Ik doe een gok" class="query__submit">
</div>
</div>
</div>
</template>
<style scoped>
.index {
text-align: center;
}
.logo {
margin-top: 230px;
margin-bottom: 30px;
}
.logo__image {
width: 472px;
}
.query__input {
margin-bottom: 20px;
}
.query__buttons > input {
background-color: #f2f2f2;
border: 1px solid #f2f2f2;
border-radius: 2px;
color: rgba(0, 0, 0, .75);
font-size: 13px;
font-weight: bold;
line-height: 27px;
margin: 11px 4px;
height: 36px;
min-width: 54px;
padding: 0 16px;
}
</style>

View File

@ -0,0 +1,202 @@
<template>
<div>
<div class="topbar">
<div class="logo">
<img class="logo__image" src="@/assets/logo.png">
</div>
<div class="query">
<input class="query__input" v-model="query" @change="getResults">
</div>
</div>
<div class="body">
<div class="result-count">
Ongeveer {{hits}} resultaten ({{responseTime.toFixed(2)}} seconden)
</div>
<div class="body__inner">
<div class="body__facets">
<div v-for="facet in facets.categories" v-bind:key="facet.category"
v-bind:class="{active: activeCategories.includes(facet.category)}"
class="category facet" @click="toggleCategory(facet)">
<div class="category__facet">{{facet.category}}</div>
<div v-if="facet.count < 10000" class="category__count">{{facet.count}}</div>
<div v-else class="category__count">{{(facet.count / 1000).toFixed()}}K</div>
</div>
</div>
<div class="body__results">
<div v-for="result in results" v-bind:key="result.id" class="result">
<a :href="result.url" target="_blank">
<div class="result__title">{{result.title}}</div>
<div class="result__link">{{result.url}}</div>
<div class="result__body">
<span class="result__date">{{result.date}} -</span>
{{result.body}}
</div>
</a>
</div>
</div>
</div>
</div>
</div>
</template>
<style scoped>
a {
text-decoration: none;
}
.topbar {
background-color: rgb(250, 250, 250);
display: flex;
flex-flow: row;
align-items: center;
}
.logo__image {
width: 180px;
margin: 20px 16px 12px;
}
.query__input {
width: 550px;
margin: 16px;
}
.body {
width: 100%;
}
.body__inner {
display: flex;
}
.result-count {
line-height: 43px;
color: rgba(0, 0, 0, 0.5);
margin-left: 208px;
}
.result {
margin: 24px 0;
max-width: 600px;
}
.result:first-child {
margin-top: 12px;
}
.result__title {
font-size: 18px;
margin: 1px 0;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.result__link {
font-size: 14px;
color: #006621;
}
.result__body {
margin: 4px 0;
color: rgba(0, 0, 0, .68);
}
.result__date {
color: rgba(0, 0, 0, .5);
}
.body__facets {
width: 184px;
margin-right: 24px;
}
.facet {
padding: 4px 8px;
cursor: pointer;
}
.facet:hover {
background-color: rgba(0, 0, 0, .1);
}
.facet.active {
font-weight: bold;
}
.category {
display: flex;
justify-content: space-between;
}
.category__facet {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
padding-right: 8px;
}
.category__count {
text-align: right;
}
</style>
<script>
import Vue from "vue";
import {Component} from "vue-property-decorator";
@Component
export default class Results extends Vue {
query = "";
hits = 0;
results = [];
facets = {};
activeCategories = [];
responseTime = 0.0;
async toggleCategory(category) {
const name = category.category;
const index = this.activeCategories.indexOf(name);
if (index === -1) {
this.activeCategories.push(name);
} else {
this.activeCategories.splice(index, 1);
}
await this.getResults();
}
async getResults() {
let url = `/api/?q=${this.query}`;
if (this.activeCategories.length > 0) {
const categories = encodeURIComponent(this.activeCategories.join(","));
url += `&categories=${categories}`;
}
let response = await fetch(url);
let json = await response.json();
this.results = json.results;
this.hits = json.hits;
this.responseTime = json.took;
this.facets = json.facets;
}
async mounted() {
await this.getResults();
}
}
</script>

12
frontend/src/main.js Normal file
View File

@ -0,0 +1,12 @@
import Vue from "vue";
import App from "./App";
import router from "./router";
Vue.config.productionTip = false;
/* eslint-disable no-new */
new Vue({
el: "#app",
router,
render: h => h(App),
});

View File

@ -0,0 +1,22 @@
import Vue from "vue";
import Router from "vue-router";
import Index from "@/components/Index";
import Results from "@/components/Results";
Vue.use(Router);
export default new Router({
routes: [
{
path: "/search",
name: "Results",
component: Results,
},
{
path: "/",
name: "Index",
component: Index,
},
],
});

0
frontend/static/.gitkeep Normal file
View File

7205
frontend/yarn.lock Normal file

File diff suppressed because it is too large Load Diff