Init commit

This commit is contained in:
Amazed 2018-10-24 23:57:41 +02:00
parent 0bfde78ace
commit 35ec3d0655
42 changed files with 38903 additions and 0 deletions

3
.gitignore vendored
View File

@ -94,3 +94,6 @@ ENV/
# Rope project settings
.ropeproject
db.sqlite3
.idea/

0
hrss/__init__.py Normal file
View File

123
hrss/settings.py Normal file
View File

@ -0,0 +1,123 @@
"""
Django settings for hrss project.
Generated by 'django-admin startproject' using Django 2.0.7.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.0/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'g!2ky&h1!r$xwxvi%nzhdoxa3z3j$zwavyrr2r&pda=#ru$2(g'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'web'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'hrss.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [os.path.join(BASE_DIR, 'web/templates/')],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'web.context_processor.BASE_URL'
],
},
},
]
WSGI_APPLICATION = 'hrss.wsgi.application'
# Database
# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/2.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.0/howto/static-files/
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

23
hrss/urls.py Normal file
View File

@ -0,0 +1,23 @@
"""hrss URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf.urls import include, url
urlpatterns = [
path('admin/', admin.site.urls),
url(r'', include('web.urls')),
]

16
hrss/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for hrss project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hrss.settings")
application = get_wsgi_application()

15
manage.py Normal file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hrss.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)

0
web/__init__.py Normal file
View File

3
web/admin.py Normal file
View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

5
web/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class WebConfig(AppConfig):
name = 'web'

9
web/context_processor.py Normal file
View File

@ -0,0 +1,9 @@
def BASE_URL(request):
"""
Return a BASE_URL template context for the current request.
"""
if request.is_secure():
scheme = 'https://'
else:
scheme = 'http://'
return {'BASE_URL': scheme + request.get_host(), }

View File

@ -0,0 +1,30 @@
# Generated by Django 2.0.7 on 2018-10-24 17:21
from django.db import migrations, models
import web.models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Feed',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(max_length=255)),
('element', models.CharField(max_length=255)),
('title', models.CharField(max_length=255)),
('content', models.CharField(max_length=255)),
('date', models.CharField(max_length=255)),
('author', models.CharField(max_length=255)),
('link', models.CharField(max_length=255)),
('creation_date', models.DateTimeField(auto_now=True)),
('uurl', models.CharField(default=web.models.random_url, max_length=18, unique=True)),
],
),
]

View File

21
web/models.py Normal file
View File

@ -0,0 +1,21 @@
from django.db import models
import uuid
# Create your models here.
def random_url():
d = uuid.uuid4()
str = d.hex
return str[0:16]
class Feed(models.Model):
url = models.URLField(max_length=255)
element = models.CharField(max_length=255)
title = models.CharField(max_length=255)
content = models.CharField(max_length=255)
date = models.CharField(max_length=255)
author = models.CharField(max_length=255)
link = models.CharField(max_length=255)
creation_date = models.DateTimeField(auto_now=True)
uurl = models.CharField(max_length=18, default=random_url, unique=True)

9030
web/static/css/bootstrap.css vendored Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

7
web/static/css/bootstrap.min.css vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,123 @@
/*!
* Start Bootstrap - Simple Sidebar (https://startbootstrap.com/template-overviews/simple-sidebar)
* Copyright 2013-2017 Start Bootstrap
* Licensed under MIT (https://github.com/BlackrockDigital/startbootstrap-simple-sidebar/blob/master/LICENSE)
*/
body {
overflow-x: hidden;
}
#wrapper {
padding-left: 0;
-webkit-transition: all 0.5s ease;
-moz-transition: all 0.5s ease;
-o-transition: all 0.5s ease;
transition: all 0.5s ease;
}
#wrapper.toggled {
padding-left: 250px;
}
#sidebar-wrapper {
z-index: 1000;
position: fixed;
left: 250px;
width: 0;
height: 100%;
margin-left: -250px;
overflow-y: auto;
background: #000;
-webkit-transition: all 0.5s ease;
-moz-transition: all 0.5s ease;
-o-transition: all 0.5s ease;
transition: all 0.5s ease;
}
#wrapper.toggled #sidebar-wrapper {
width: 250px;
}
#page-content-wrapper {
width: 100%;
position: absolute;
padding: 15px;
}
#wrapper.toggled #page-content-wrapper {
position: absolute;
margin-right: -250px;
}
/* Sidebar Styles */
.sidebar-nav {
position: absolute;
top: 0;
width: 250px;
margin: 0;
padding: 0;
list-style: none;
}
.sidebar-nav li {
/*text-indent: 20px;*/
padding-left:20px;
line-height: 40px;
}
.sidebar-nav li a {
display: block;
text-decoration: none;
color: #999999;
}
.sidebar-nav li a:hover {
text-decoration: none;
color: #fff;
background: rgba(255, 255, 255, 0.2);
}
.sidebar-nav li a:active, .sidebar-nav li a:focus {
text-decoration: none;
}
.sidebar-nav>.sidebar-brand {
height: 65px;
font-size: 18px;
line-height: 60px;
}
.sidebar-nav>.sidebar-brand a {
color: #999999;
}
.sidebar-nav>.sidebar-brand a:hover {
color: #fff;
background: none;
}
@media(min-width:768px) {
#wrapper {
padding-left: 0;
}
#wrapper.toggled {
padding-left: 250px;
}
#sidebar-wrapper {
width: 0;
}
#wrapper.toggled #sidebar-wrapper {
width: 250px;
}
#page-content-wrapper {
padding: 20px;
position: relative;
}
#wrapper.toggled #page-content-wrapper {
position: relative;
margin-right: 0;
}
}

6461
web/static/js/bootstrap.bundle.js vendored Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

7
web/static/js/bootstrap.bundle.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

3944
web/static/js/bootstrap.js vendored Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

7
web/static/js/bootstrap.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

29
web/static/js/jquery-getpath.js vendored Normal file
View File

@ -0,0 +1,29 @@
jQuery.fn.extend({
getPath: function( path ) {
// The first time this function is called, path won't be defined.
if ( typeof path == 'undefined' ) path = '';
// If this element is <html> we've reached the end of the path.
if ( this.is('html') )
return 'html' + path;
// Add the element name.
var cur = this[0].nodeName.toLowerCase();
// Determine the IDs and path.
var id = this.attr('id');
var cls = this.attr('class');
// Add the #id if there is one.
if ( typeof id != 'undefined' )
cur += '#' + id;
// Add any classes.
if ( typeof cls != 'undefined' )
cur += '.' + cls.split(/[\s\n]+/).join('.');
// Recurse up the DOM.
return this.parent().getPath( ' > ' + cur + path );
}
});

10364
web/static/js/jquery.js vendored Normal file

File diff suppressed because it is too large Load Diff

2
web/static/js/jquery.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

8269
web/static/js/jquery.slim.js Normal file

File diff suppressed because it is too large Load Diff

2
web/static/js/jquery.slim.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

18
web/templates/feed.xml Normal file
View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>{{ rss.title }}</title>
<icon>https://www.google.com/s2/favicons?domain={{ feed.url }}</icon>
<logo>https://www.google.com/s2/favicons?domain={{ feed.url }}</logo>
<description>Generated by HRSS</description>
<link>{{ feed.url }}</link>
{% for item in rss.items %}
<item>
<title>{{ item.title }}</title>
<description>{{ item.content }}</description>
<pubDate>{{ item.date }}</pubDate>
<link>{{ item.link }}</link>
</item>
{% endfor %}
</channel>
</rss>

33
web/templates/feeds.html Normal file
View File

@ -0,0 +1,33 @@
{% extends 'template.html' %}
{% block content %}
<div class="container">
<div class="row">
<div class="col-lg-12 text-center">
<h1 class="mt-5">Feeds</h1>
{% if error %}<div class="alert alert-danger" role="alert">{{ error }}</div>{% endif %}
{% if feeds %}
<div class="accordion" id="accordionExample">
{% for feed in feeds %}
<div class="card">
<div class="card-header" id="heading{{ feed.id }}">
<h5 class="mb-0">
<button class="btn btn-link" type="button" data-toggle="collapse" data-target="#collapse{{ feed.id }}" aria-expanded="true" aria-controls="collapse{{ feed.id }}">
<img src="https://www.google.com/s2/favicons?domain={{ feed.url }}"> {{ feed.url }}
</button>
</h5>
</div>
<div id="collapse{{ feed.id }}" class="collapse" aria-labelledby="heading{{ feed.id }}" data-parent="#accordionExample">
<div class="card-body">
<code>{{ BASE_URL }}/{{ feed.uurl }}.rss</code>
<p><a class="btn btn-danger" href="{% url 'feed_delete' feed.id %}">Delete</a></p>
</div>
</div>
</div>
{% endfor %}
</div>
{% endif %}
</div>
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,18 @@
{% extends 'template.html' %}
{% block content %}
<div class="container">
<div class="row">
<div class="col-lg-12 text-center">
<h1 class="mt-5">Generate RSS out of any website</h1>
{% if error %}<div class="alert alert-danger" role="alert">{{ error }}</div>{% endif %}
<form action="/" method="post">
{% csrf_token %}
<div class="form-group">
<input type="text" class="form-control" name="url" id="url" placeholder="http://" {% if url %}value="{{ url }}"{% endif %}>
</div>
<button type="submit" class="btn btn-primary">Setup</button>
</form>
</div>
</div>
</div>
{% endblock %}

117
web/templates/setup.html Normal file
View File

@ -0,0 +1,117 @@
{% extends 'template.html' %}
{% block content %}
<div id="wrapper" class="toggled bg-dark">
<!-- Sidebar -->
<div id="sidebar-wrapper" class="bg-dark bg-">
<ul class="sidebar-nav">
<li>
<span class="badge badge-danger">Element</span>
<code style="display:block" id="element-selector"></code>
</li>
<li>
<span class="badge badge-danger">Title</span>
<code style="display:block" id="title-selector"></code>
</li>
<li>
<span class="badge badge-danger">Content</span>
<code style="display:block" id="content-selector"></code>
</li>
<li>
<span class="badge badge-danger">Date</span>
<code style="display:block" id="date-selector"></code>
</li>
<li>
<span class="badge badge-danger">Author</span>
<code style="display:block" id="author-selector"></code>
</li>
<li>
<span class="badge badge-danger">Link</span>
<code style="display:block" id="link-selector"></code>
</li>
<li>
<form action="/newfeed" method="post">
{% csrf_token %}
<input type="hidden" id="url" name="url" value="{{ url }}">
<input type="hidden" id="element" name="element" value="">
<input type="hidden" id="title" name="title" value="">
<input type="hidden" id="content" name="content" value="">
<input type="hidden" id="date" name="date" value="">
<input type="hidden" id="author" name="author" value="">
<input type="hidden" id="link" name="link" value="">
<input type="submit" class="btn btn-primary" value="Generate">
</form>
</li>
</ul>
</div>
<!-- /#sidebar-wrapper -->
<iframe id="preview" style="width:100%" src="/iframe/{{ url }}"></iframe>
<script>
$(function() {
function handleResize() {
$("#preview").height($(window).height());
}
$(window).resize(function() {
handleResize();
});
handleResize();
});
var lastCSS = false;
function hoverSelector() {
lastCSS = $(lastElement).css("border");
$(lastElement).css("border", "2px dashed orange")
}
function resetHoverSelector() {
$(lastElement).css("border", lastCSS);
}
var lastElement = false;
var selectedElement = false;
$("iframe").on("load", function() {
var iframeWnd = $($("iframe")[0].contentWindow);
// disable a links
$("iframe").contents().find("a").attr("href", "#");
iframeWnd.on('click', function(event) {
selectedElement = $(event.target).getPath();
if (currentProperty) {
$("#"+$(currentProperty).text().toLowerCase()).val(selectedElement);
var t = selectedElement.split(">");
$("#"+$(currentProperty).text().toLowerCase()+"-selector").text(t[t.length-1]);
// reset
$(".sidebar-nav > li > span.badge-warning").removeClass("badge-warning").addClass("badge-success");
currentProperty = false;
resetHoverSelector();
}
});
iframeWnd.on('mousemove', function(event) {
if (currentProperty && lastElement != event.target) {
resetHoverSelector();
lastElement = event.target;
hoverSelector();
}
});
});
var currentProperty = false;
$(".sidebar-nav > li > span.badge").click(function(evt) {
var selectedProperty = $(this).text().toLowerCase();
// reset every elements
$(".sidebar-nav > li > span.badge.badge-warning").removeClass("badge-warning").addClass("badge-danger");
//$(".sidebar-nav > li > span.badge");
// set element
$(this).removeClass("badge-danger");
$(this).addClass("badge-warning");
currentProperty = this;
});
</script>
</div>
{% endblock %}

View File

@ -0,0 +1,56 @@
{% load static %}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="">
<meta name="author" content="">
<title>HRSS</title>
<!-- Bootstrap core CSS -->
<link href="{% static 'css/bootstrap.css' %}" rel="stylesheet">
<link href="{% static 'css/simple-sidebar.css' %}" rel="stylesheet">
<script src="{% static 'js/jquery.min.js' %}"></script>
<script src="{% static 'js/jquery-getpath.js' %}"></script>
</head>
<body>
<!-- Navigation -->
<nav class="navbar navbar-expand-lg navbar-dark bg-dark static-top">
<div class="container">
<a class="navbar-brand" href="/">HRSS</a>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarResponsive" aria-controls="navbarResponsive" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarResponsive">
<ul class="navbar-nav ml-auto">
<li class="nav-item">
<a class="nav-link" href="/">New
<span class="sr-only">(current)</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{% url 'feeds' %}">Feeds</a>
</li>
</ul>
</div>
</div>
</nav>
{% block content %}
{% endblock %}
<!-- Bootstrap core JavaScript -->
<script src="{% static 'js/bootstrap.bundle.min.js' %}"></script>
</body>
</html>

3
web/tests.py Normal file
View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

12
web/urls.py Normal file
View File

@ -0,0 +1,12 @@
from django.conf.urls import url
from . import views
urlpatterns = [
url(r'^$', views.homepage, name='homepage'),
url(r'^iframe/(?P<url>.+)$', views.iframe, name='iframe'),
url(r'^setup/(?P<url>.+)$', views.setup, name='setup'),
url(r'^newfeed$', views.newfeed, name='newfeed'),
url(r'^feeds$', views.feeds, name='feeds'),
url(r'^feeds/delete/(?P<id>[0-9]+)$', views.feed_delete, name='feed_delete'),
url(r'^(?P<uurl>.+).rss$', views.rss, name='rss'),
]

58
web/utils.py Normal file
View File

@ -0,0 +1,58 @@
import requests
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
def is_valid_url(url):
try:
URLValidator()(url)
requests.get(url).raise_for_status()
return True
except ValidationError as e:
print(url+" is not a valid url")
return False
except RequestException as e:
print(url+" led to exception: "+str(e))
return False
def find_longest_common_string(s1, s2):
from difflib import SequenceMatcher
match = SequenceMatcher(None, s1, s2).find_longest_match(0, len(s1), 0, len(s2))
return s1[match.a: match.a + match.size] # -> apple pie
def fetch_feed(feed, limit=10):
items=[]
rep = requests.get(feed.url).text
soup = BeautifulSoup(rep)
elements = soup.select(feed.element)
for element in elements:
if len(items) >= limit:
break
try:
title = element.select(feed.title)[0].text
except Exception:
return False
try:
content = element.select(feed.content)[0].text
except Exception:
return False
try:
date = element.select(feed.date)[0].text
except Exception:
date = False
try:
author = element.select(feed.author)[0].text
except Exception:
author = False
try:
link = element.select(feed.link)[0]["href"]
except Exception:
link = False
items.append({"title": title, "content": content, "pubDate": date, "author": author, "link": link})
return {"title": soup.find("title").text, "items": items}

87
web/views.py Normal file
View File

@ -0,0 +1,87 @@
from django.shortcuts import render, redirect
from django.http import HttpResponse
from .utils import *
from .models import Feed
from django.db.models import ObjectDoesNotExist
# Create your views here.
def iframe(request, url):
try:
html = requests.get(url).text
except Exception as e:
html = str(e)
return HttpResponse(html)
def dummy(request):
return HttpResponse("toto")
def homepage(request):
if request.method == 'POST':
if "url" in request.POST and request.POST["url"]:
url = request.POST["url"]
if is_valid_url(url):
return redirect("/setup/"+url)
else:
return render(request, 'homepage.html', {"url": url, "error": url+" is not a valid URL."})
return render(request, 'homepage.html')
def setup(request, url):
if is_valid_url(url):
return render(request, 'setup.html', {"url": url})
else:
return redirect("/")
def newfeed(request):
if request.method == 'POST':
if not "url" in request.POST or not "element" in request.POST or not "title" in request.POST or not "content" in request.POST:
return HttpResponse("Error, missing required element")
url = request.POST["url"]
element = request.POST["element"]
title = request.POST["title"]
content = request.POST["content"]
date = request.POST["date"]
author = request.POST["author"]
link = request.POST["link"]
a = find_longest_common_string(element, title)
title = title[len(a)+3:].strip()
a = find_longest_common_string(element, content)
content = content[len(a)+3:].strip()
a = find_longest_common_string(element, date)
date = date[len(a)+3:].strip()
a = find_longest_common_string(element, author)
author = author[len(a)+3:].strip()
a = find_longest_common_string(element, link)
link = link[len(a)+3:].strip()
feed = Feed(url=url, element=element, title=title, content=content, date=date, author=author, link=link)
feed.save()
return redirect("/feeds")
else:
return redirect("/")
def feeds(request):
feeds = Feed.objects.all()
return render(request, "feeds.html", {"feeds": feeds})
def feed_delete(request, id):
try:
Feed.objects.get(pk=id).delete()
except Exception as e:
pass
finally:
return redirect("/feeds")
def rss(request, uurl):
try:
feed = Feed.objects.get(uurl=uurl)
return render(request, "feed.xml", {"feed": feed, "rss": fetch_feed(feed)}, content_type="application/rss+xml")
except ObjectDoesNotExist:
return HttpResponse("nope")