diff --git a/.gitignore b/.gitignore index 6102faf..7c95164 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea/ __pycache__ *~ +testenv/tests/webpage/cgitestrun/cgi-bin +testenv/tests/webpage/cgitestrun/testfeeds \ No newline at end of file diff --git a/src/config/config.py b/src/config/config.py index f90e6e7..a42feaa 100644 --- a/src/config/config.py +++ b/src/config/config.py @@ -1,10 +1,11 @@ __author__ = 'dev' Config = { - "FeedLinkTypes" : {"audio/mpeg","application/x-bittorrent"}, + "FeedLinkTypes" : {"audio/mpeg","application/x-bittorrent", "audio/opus"}, "FeedTypes" : {"podcast","rss"}, "FeedStorageFolder" : "testfeeds", "FeedStoragePrefix" : { "0" : "FeedType", "1" : "FeedName"}, "ConfigFileSuffix" : "-config", - "ConfigFileExtension" : ".json" + "ConfigFileExtension" : ".json", + "TorrentStorageFolder": "testtorrents", } \ No newline at end of file diff --git a/src/feed/entry_handler.py b/src/feed/entry_handler.py new file mode 100644 index 0000000..e99e2be --- /dev/null +++ b/src/feed/entry_handler.py @@ -0,0 +1,28 @@ +__author__ = 'dev' + +from feed.feed import Feed +import utilities.file_methods as fm +from feed.feed_entry_enums import FeedEntryEnums + + +class EntryHandler(): + + def __init__(self, feed): + assert isinstance(feed, Feed) + self._feed = feed + + def handle_entries(self): + for entry in self.detect_new_entries(): + self.handle_entry(entry) + + def detect_new_entries(self): + new_links = [] + for entry in fm.read_json_file(self._feed.feed_file_path)[FeedEntryEnums.KeyName.EntryKey]: + status = entry.get(FeedEntryEnums.KeyName.EntryStatusKey) + if status is None or status == FeedEntryEnums.Status.New: + new_links.append(entry) + + return new_links + + def handle_entry(self, link): + pass \ No newline at end of file diff --git a/src/feed/feed.py b/src/feed/feed.py index 9495aa8..28578a0 100644 --- a/src/feed/feed.py +++ b/src/feed/feed.py @@ -2,8 +2,10 @@ __author__ = 'dev' from config.config import Config from utilities.podcastserializer import PodcastSerializer +from utilities.serializer import Serializer import feedparser import os +import glob import utilities.file_methods as fm @@ -22,6 +24,8 @@ class Feed: self.feed_file_path = None self.serializer = None self.feed_folder = None + self.name = feed_name + self.type = feed_type self.setup_feed(feed_name, feed_type, link_type, feed_url) def load_feed(self): @@ -62,8 +66,9 @@ class Feed: :return: Serializer - the serializer of the feed """ return { - "podcast": PodcastSerializer(self.feed_config["LinkType"]) - }.get(self.feed_config["FeedType"], PodcastSerializer(self.feed_config["LinkType"])) + "podcast": PodcastSerializer(self.feed_config["LinkType"]), + "rss": Serializer() + }.get(self.feed_config["FeedType"], Serializer()) def setup_feed(self, feed_name, feed_type, link_type, feed_url): """ @@ -110,7 +115,7 @@ class Feed: self.feed_folder_path = self.feed_folder if not os.path.exists(self.feed_folder_path): - os.mkdir(self.feed_folder) + os.mkdir(self.feed_folder_path) return self.feed_folder_path @@ -131,6 +136,9 @@ class Feed: assert (self.feed_config is not None) fm.write_json_file(self.create_feed_config_path(), self.feed_config) + def get_feed_items(self): + return fm.read_json_file(self.feed_file_path) + @staticmethod def create_feed_config_file(feed_name, feed_type, link_type, feed_url): """ @@ -153,4 +161,51 @@ class Feed: "FeedUrl": feed_url, "LinkType": link_type } - return feed_config \ No newline at end of file + return feed_config + + @staticmethod + def create_feed_from_feed_config(feed_config): + assert feed_config is not None + feed_name = feed_config["FeedName"] + feed_type = feed_config["FeedType"] + feed_url = feed_config["FeedUrl"] + link_type = feed_config["LinkType"] + return Feed(feed_name, feed_type, link_type, feed_url) + + @staticmethod + def create_feed_from_feed_config_path(feed_config_path): + assert os.path.isfile(feed_config_path) + feed_config = fm.read_json_file(feed_config_path) + return Feed.create_feed_from_feed_config(feed_config) + + @staticmethod + def create_feed_from_feed_folder_path(feed_folder_path): + assert os.path.isdir(feed_folder_path) + feed_config_path = glob.glob(feed_folder_path + "/*" + Config["ConfigFileSuffix"] + Config["ConfigFileExtension"])[0] + return Feed.create_feed_from_feed_config_path(feed_config_path) + + @staticmethod + def create_feeds_from_storage_folder(): + feed_list_items = [] + for file_path in glob.glob(Config["FeedStorageFolder"] + "/*" * (len(Config["FeedStoragePrefix"]) + 1) + Config["ConfigFileSuffix"] + Config["ConfigFileExtension"], recursive=True): + try: + feed = Feed.create_feed_from_feed_config_path(file_path) + feed_list_items.append(feed) + except: + pass + return feed_list_items + + @staticmethod + def FindItem(feed_items, item_key, search_pattern): + assert feed_items is not None + assert item_key is not None + assert search_pattern is not None + result = None + for feed_item in feed_items: + if item_key not in feed_item: + continue + if feed_item[item_key] == search_pattern: + result = feed_item + break + + return result \ No newline at end of file diff --git a/src/feed/feed_entry_enums.py b/src/feed/feed_entry_enums.py new file mode 100644 index 0000000..d20e969 --- /dev/null +++ b/src/feed/feed_entry_enums.py @@ -0,0 +1,12 @@ +__author__ = 'dev' + + +class FeedEntryEnums: + class Status: + New = "New" + Loading = "Loading" + Loaded = "Loaded" + + class KeyName: + EntryKey = "entries" + EntryStatusKey = "EntryStatus" \ No newline at end of file diff --git a/src/feed/torrent_cli_entry_handler.py b/src/feed/torrent_cli_entry_handler.py new file mode 100644 index 0000000..12ab971 --- /dev/null +++ b/src/feed/torrent_cli_entry_handler.py @@ -0,0 +1,110 @@ +__author__ = 'dev' + +from feed.entry_handler import EntryHandler +import urllib +import os +import shutil +from config.config import Config +import utilities.file_methods as fm +import subprocess + + +class TorrentCliEntryHandler(EntryHandler): + + Transmission_Settings = { + "alt-speed-down": 50, + "alt-speed-enabled": False, + "alt-speed-time-begin": 540, + "alt-speed-time-day": 127, + "alt-speed-time-enabled": False, + "alt-speed-time-end": 1020, + "alt-speed-up": 50, + "bind-address-ipv4": "0.0.0.0", + "bind-address-ipv6": "::", + "blocklist-enabled": False, + "blocklist-url": "http://www.example.com/blocklist", + "cache-size-mb": 4, + "dht-enabled": True, + "download-dir": "tmp", + "download-queue-enabled": True, + "download-queue-size": 5, + "encryption": 2, + "idle-seeding-limit": 30, + "idle-seeding-limit-enabled": False, + "incomplete-dir": "/home/dev/Downloads", + "incomplete-dir-enabled": False, + "lpd-enabled": False, + "message-level": 2, + "peer-congestion-algorithm": "", + "peer-id-ttl-hours": 6, + "peer-limit-global": 200, + "peer-limit-per-torrent": 50, + "peer-port": 51413, + "peer-port-random-high": 65535, + "peer-port-random-low": 49152, + "peer-port-random-on-start": False, + "peer-socket-tos": "default", + "pex-enabled": True, + "port-forwarding-enabled": True, + "preallocation": 1, + "prefetch-enabled": 1, + "queue-stalled-enabled": True, + "queue-stalled-minutes": 30, + "ratio-limit": 2, + "ratio-limit-enabled": False, + "rename-partial-files": True, + "rpc-authentication-required": False, + "rpc-bind-address": "0.0.0.0", + "rpc-enabled": False, + "rpc-password": "{7788eba16f883ebbda75ba83697c6db606a86ad1aMLMCwny", + "rpc-port": 9091, + "rpc-url": "/transmission/", + "rpc-username": "", + "rpc-whitelist": "127.0.0.1", + "rpc-whitelist-enabled": True, + "scrape-paused-torrents-enabled": True, + "script-torrent-done-enabled": True, + "script-torrent-done-filename": "", + "seed-queue-enabled": False, + "seed-queue-size": 10, + "speed-limit-down": 100, + "speed-limit-down-enabled": False, + "speed-limit-up": 100, + "speed-limit-up-enabled": False, + "start-added-torrents": True, + "trash-original-torrent-files": True, + "umask": 18, + "upload-slots-per-torrent": 14, + "utp-enabled": True +} + SettingFile = Config["TorrentStorageFolder"] + "/settings.json" + + @staticmethod + def download_torrent(torrent_url): + torrent_folder = Config["TorrentStorageFolder"] + if not os.path.exists(torrent_folder): + TorrentCliEntryHandler.init_torrent_folder(torrent_folder) + file_name = torrent_folder + "/" + os.path.basename(torrent_url) + + if os.path.exists(torrent_url): + shutil.copy2(torrent_url, file_name) + + else: + with urllib.request.urlopen(torrent_url) as response, open(file_name, 'wb') as out_file: + data = response.read() + out_file.write(data) + + @staticmethod + def init_torrent_folder(torrent_folder): + if os.path.exists(torrent_folder): + return + os.mkdir(torrent_folder) + TorrentCliEntryHandler.Transmission_Settings["incomplete-dir"] = torrent_folder + "/Downloads" + TorrentCliEntryHandler.Transmission_Settings["download-dir"] = torrent_folder + "/tmp" + fm.write_json_file(TorrentCliEntryHandler.SettingFile, TorrentCliEntryHandler.Transmission_Settings) + + @staticmethod + def start_torrent(torrent_url, torrent_folder, download_folder): + #pass + tc = subprocess.Popen("transmission-cli -er -g" + torrent_folder + " -w " + download_folder + " " + torrent_url, shell=True) + return tc \ No newline at end of file diff --git a/src/feed/torrent_daemon_entry_handler.py b/src/feed/torrent_daemon_entry_handler.py new file mode 100644 index 0000000..8e2edb7 --- /dev/null +++ b/src/feed/torrent_daemon_entry_handler.py @@ -0,0 +1,38 @@ +__author__ = 'dev' + +from feed.entry_handler import EntryHandler +import subprocess +import time + +class TorrentDaemonEntryHandler(EntryHandler): + + @staticmethod + def start_daemon(): + if not TorrentDaemonEntryHandler.is_running_daemon(): + tc = subprocess.call("transmission-daemon", shell=True) + time.sleep(1) + return tc + else: + return None + + @staticmethod + def stop_daemon(): + tc = subprocess.call("killall transmission-daemon", shell=True) + time.sleep(1) + return tc + + @staticmethod + def is_running_daemon(): + tc = subprocess.Popen("pgrep transmission-da", shell=True, stdout=subprocess.PIPE) + tc.wait() + t = tc.stdout.readline() + + if t is b'': + return False + else: + return True + + @staticmethod + def add_torrent_to_daemon(url): + tc = subprocess.call("transmission-remote --add " + url, shell=True) + return tc \ No newline at end of file diff --git a/src/ui/__init__.py b/src/ui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ui/web/__init__.py b/src/ui/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ui/web/cgi/__init__.py b/src/ui/web/cgi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ui/web/cgi/cgi_server.py b/src/ui/web/cgi/cgi_server.py new file mode 100644 index 0000000..1fdef61 --- /dev/null +++ b/src/ui/web/cgi/cgi_server.py @@ -0,0 +1,21 @@ +import os +import sys +import shutil +import glob +import stat +from http.server import HTTPServer, CGIHTTPRequestHandler +import ui.web.cgi.pages + +def runServer(workingDir, serverAddress=('',8080)): + + page_folder = os.path.join(workingDir, "cgi-bin/") + if(not os.path.exists(page_folder)): + srcpagefolder = str(sys.modules['ui.web.cgi.pages']).replace("","") + shutil.copytree(srcpagefolder, page_folder) + os.chdir(page_folder) + for file in glob.glob('*.py'): + os.chmod(file, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) + + os.chdir('..') + server = HTTPServer(serverAddress, CGIHTTPRequestHandler) + server.serve_forever() \ No newline at end of file diff --git a/src/ui/web/cgi/pages/FeedOverview.py b/src/ui/web/cgi/pages/FeedOverview.py new file mode 100644 index 0000000..0691dd5 --- /dev/null +++ b/src/ui/web/cgi/pages/FeedOverview.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# enable debugging +import cgi +import urllib +from feed.feed import Feed + + +def create_episode_list(episodes, feedFolder): + result = [] + for episode in episodes: + result.append('''
  • {EpisodeTitle} {EpisodeDate} +

    {EpisodeSubTitle}

    • '''.format(EpisodeTitle=episode["title"], EpisodeSubTitle=episode["subtitle"], EpisodeDate=episode["pubdate"], EpisodeUrl=urllib.parse.urlencode({'EpisodeTitle': episode["title"], 'FeedFolder': feedFolder}))) + + return "\n".join(result) + +arguments = cgi.FieldStorage() +feedFolder =arguments["FeedFolder"].value +feed = Feed.create_feed_from_feed_folder_path(feedFolder) +feedObj = feed.get_feed_items() + +page = '''Content-Type: text/html + + + + + +Feed: {FeedName} - {FeedTitle} + + + +

    Feed: {FeedName} - {FeedTitle}

    +

    {FeedSubTitle}

    +

    {FeedSummary}

    + + + + +'''.format(FeedName=feed.name, FeedTitle=feedObj["title"], FeedSubTitle=feedObj["subtitle"], FeedSummary=feedObj["summary"], EpisodeList=create_episode_list(feedObj["episodes"], feed.feed_folder_path)) + +print(page) \ No newline at end of file diff --git a/src/ui/web/cgi/pages/FileLoader.py b/src/ui/web/cgi/pages/FileLoader.py new file mode 100644 index 0000000..8d0267f --- /dev/null +++ b/src/ui/web/cgi/pages/FileLoader.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# enable debugging +import cgi +import os +import subprocess +from feed.feed import Feed + +arguments = cgi.FieldStorage() +feedFolder = arguments["FeedFolder"].value +feed = Feed.create_feed_from_feed_folder_path(feedFolder) +feedObj = feed.get_feed_items() + +feedName = feed.name +episodeTitle = arguments["EpisodeTitle"].value +episode = Feed.FindItem(feedObj["episodes"], "title", episodeTitle) +originLink = episode["file_link"] +filename, file_extension = os.path.splitext(originLink) +mylinkPrefix = os.path.join(feedFolder, episodeTitle.replace(" ", "_")) +mylink = mylinkPrefix + file_extension +if not os.path.exists(mylink): + subprocess.call(['curl -Lo "{Mylink}" "{OriginLink}" &'.format(Mylink=mylink, OriginLink=originLink)], + shell=True + ) + + +page = '''Content-Type: text/html + + + + + +FileLoad + + +

    FileLoad

    + +

    {FeedName}: {EpisodeTitle}

    + +