Clean up formatting and help linter calm down

This commit is contained in:
Agatha Rose 2021-06-05 00:38:36 +03:00
parent dd78364f2d
commit a904587b32
No known key found for this signature in database
GPG key ID: 2DB18BA2E0A80BC3
4 changed files with 92 additions and 81 deletions

View file

@ -8,12 +8,13 @@ from bs4 import BeautifulSoup
from random import randint from random import randint
import re, multiprocessing, sqlite3, shutil, os, html import re, multiprocessing, sqlite3, shutil, os, html
def make_sentence(output, cfg):
class nlt_fixed(markovify.NewlineText): #modified version of NewlineText that never rejects sentences
def test_sentence_input(self, sentence):
return True #all sentences are valid <3
shutil.copyfile("toots.db", "toots-copy.db") #create a copy of the database because reply.py will be using the main one def make_sentence(output, cfg):
class nlt_fixed(markovify.NewlineText): # modified version of NewlineText that never rejects sentences
def test_sentence_input(self, sentence):
return True # all sentences are valid <3
shutil.copyfile("toots.db", "toots-copy.db") # create a copy of the database because reply.py will be using the main one
db = sqlite3.connect("toots-copy.db") db = sqlite3.connect("toots-copy.db")
db.text_factory = str db.text_factory = str
c = db.cursor() c = db.cursor()
@ -35,8 +36,6 @@ def make_sentence(output, cfg):
db.close() db.close()
os.remove("toots-copy.db") os.remove("toots-copy.db")
toots_str = None
if cfg['limit_length']: if cfg['limit_length']:
sentence_len = randint(cfg['length_lower_limit'], cfg['length_upper_limit']) sentence_len = randint(cfg['length_lower_limit'], cfg['length_upper_limit'])
@ -59,22 +58,24 @@ def make_sentence(output, cfg):
output.send(sentence) output.send(sentence)
def make_toot(cfg): def make_toot(cfg):
toot = None toot = None
pin, pout = multiprocessing.Pipe(False) pin, pout = multiprocessing.Pipe(False)
p = multiprocessing.Process(target = make_sentence, args = [pout, cfg]) p = multiprocessing.Process(target=make_sentence, args=[pout, cfg])
p.start() p.start()
p.join(5) #wait 5 seconds to get something p.join(5) # wait 5 seconds to get something
if p.is_alive(): #if it's still trying to make a toot after 5 seconds if p.is_alive(): # if it's still trying to make a toot after 5 seconds
p.terminate() p.terminate()
p.join() p.join()
else: else:
toot = pin.recv() toot = pin.recv()
if toot == None: if toot is None:
toot = "Toot generation failed! Contact Lynne (lynnesbian@fedi.lynnesbian.space) for assistance." toot = "Toot generation failed! Contact Lynne (lynnesbian@fedi.lynnesbian.space) for assistance."
return toot return toot
def extract_toot(toot): def extract_toot(toot):
toot = html.unescape(toot) # convert HTML escape codes to text toot = html.unescape(toot) # convert HTML escape codes to text
soup = BeautifulSoup(toot, "html.parser") soup = BeautifulSoup(toot, "html.parser")
@ -87,7 +88,7 @@ def extract_toot(toot):
for ht in soup.select("a.hashtag"): # convert hashtags from links to text for ht in soup.select("a.hashtag"): # convert hashtags from links to text
ht.unwrap() ht.unwrap()
for link in soup.select("a"): #ocnvert <a href='https://example.com>example.com</a> to just https://example.com for link in soup.select("a"): # convert <a href='https://example.com>example.com</a> to just https://example.com
if 'href' in link: if 'href' in link:
# apparently not all a tags have a href, which is understandable if you're doing normal web stuff, but on a social media platform?? # apparently not all a tags have a href, which is understandable if you're doing normal web stuff, but on a social media platform??
link.replace_with(link["href"]) link.replace_with(link["href"])

12
gen.py
View file

@ -8,9 +8,11 @@ import argparse, json, re
import functions import functions
parser = argparse.ArgumentParser(description='Generate and post a toot.') parser = argparse.ArgumentParser(description='Generate and post a toot.')
parser.add_argument('-c', '--cfg', dest='cfg', default='config.json', nargs='?', parser.add_argument(
'-c', '--cfg', dest='cfg', default='config.json', nargs='?',
help="Specify a custom location for config.json.") help="Specify a custom location for config.json.")
parser.add_argument('-s', '--simulate', dest='simulate', action='store_true', parser.add_argument(
'-s', '--simulate', dest='simulate', action='store_true',
help="Print the toot without actually posting it. Use this to make sure your bot's actually working.") help="Print the toot without actually posting it. Use this to make sure your bot's actually working.")
args = parser.parse_args() args = parser.parse_args()
@ -32,10 +34,10 @@ if __name__ == '__main__':
toot = re.sub(r"[\[\]\(\)\{\}\"“”«»„]", "", toot) toot = re.sub(r"[\[\]\(\)\{\}\"“”«»„]", "", toot)
if not args.simulate: if not args.simulate:
try: try:
client.status_post(toot, visibility = 'unlisted', spoiler_text = cfg['cw']) client.status_post(toot, visibility='unlisted', spoiler_text=cfg['cw'])
except Exception as err: except Exception:
toot = "An error occurred while submitting the generated post. Contact lynnesbian@fedi.lynnesbian.space for assistance." toot = "An error occurred while submitting the generated post. Contact lynnesbian@fedi.lynnesbian.space for assistance."
client.status_post(toot, visibility = 'unlisted', spoiler_text = "Error!") client.status_post(toot, visibility='unlisted', spoiler_text="Error!")
try: try:
print(toot) print(toot)
except UnicodeEncodeError: except UnicodeEncodeError:

53
main.py
View file

@ -5,20 +5,19 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/. # file, You can obtain one at http://mozilla.org/MPL/2.0/.
from mastodon import Mastodon, MastodonUnauthorizedError from mastodon import Mastodon, MastodonUnauthorizedError
from os import path import sqlite3, signal, sys, json, re, argparse
from bs4 import BeautifulSoup
import os, sqlite3, signal, sys, json, re, shutil, argparse
import requests import requests
import functions import functions
parser = argparse.ArgumentParser(description='Log in and download posts.') parser = argparse.ArgumentParser(description='Log in and download posts.')
parser.add_argument('-c', '--cfg', dest='cfg', default='config.json', nargs='?', parser.add_argument(
'-c', '--cfg', dest='cfg', default='config.json', nargs='?',
help="Specify a custom location for config.json.") help="Specify a custom location for config.json.")
args = parser.parse_args() args = parser.parse_args()
scopes = ["read:statuses", "read:accounts", "read:follows", "write:statuses", "read:notifications", "write:accounts"] scopes = ["read:statuses", "read:accounts", "read:follows", "write:statuses", "read:notifications", "write:accounts"]
#cfg defaults # cfg defaults
cfg = { cfg = {
"site": "https://botsin.space", "site": "https://botsin.space",
@ -48,7 +47,8 @@ if not cfg['site'].startswith("https://") and not cfg['site'].startswith("http:/
if "client" not in cfg: if "client" not in cfg:
print("No application info -- registering application with {}".format(cfg['site'])) print("No application info -- registering application with {}".format(cfg['site']))
client_id, client_secret = Mastodon.create_app("mstdn-ebooks", client_id, client_secret = Mastodon.create_app(
"mstdn-ebooks",
api_base_url=cfg['site'], api_base_url=cfg['site'],
scopes=scopes, scopes=scopes,
website="https://github.com/Lynnesbian/mstdn-ebooks") website="https://github.com/Lynnesbian/mstdn-ebooks")
@ -60,8 +60,9 @@ if "client" not in cfg:
if "secret" not in cfg: if "secret" not in cfg:
print("No user credentials -- logging in to {}".format(cfg['site'])) print("No user credentials -- logging in to {}".format(cfg['site']))
client = Mastodon(client_id = cfg['client']['id'], client = Mastodon(
client_secret = cfg['client']['secret'], client_id=cfg['client']['id'],
client_secret=cfg['client']['secret'],
api_base_url=cfg['site']) api_base_url=cfg['site'])
print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes))) print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes)))
@ -69,14 +70,16 @@ if "secret" not in cfg:
json.dump(cfg, open(args.cfg, "w+")) json.dump(cfg, open(args.cfg, "w+"))
def extract_toot(toot): def extract_toot(toot):
toot = functions.extract_toot(toot) toot = functions.extract_toot(toot)
toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning toot = toot.replace("@", "@\u200B") # put a zws between @ and username to avoid mentioning
return(toot) return(toot)
client = Mastodon( client = Mastodon(
client_id=cfg['client']['id'], client_id=cfg['client']['id'],
client_secret = cfg['client']['secret'], client_secret=cfg['client']['secret'],
access_token=cfg['secret'], access_token=cfg['secret'],
api_base_url=cfg['site']) api_base_url=cfg['site'])
@ -89,7 +92,7 @@ except MastodonUnauthorizedError:
following = client.account_following(me.id) following = client.account_following(me.id)
db = sqlite3.connect("toots.db") db = sqlite3.connect("toots.db")
db.text_factory=str db.text_factory = str
c = db.cursor() c = db.cursor()
c.execute("CREATE TABLE IF NOT EXISTS `toots` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)") c.execute("CREATE TABLE IF NOT EXISTS `toots` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)")
c.execute("CREATE TRIGGER IF NOT EXISTS `dedup` AFTER INSERT ON toots FOR EACH ROW BEGIN DELETE FROM toots WHERE rowid NOT IN (SELECT MIN(sortid) FROM toots GROUP BY uri ); END; ") c.execute("CREATE TRIGGER IF NOT EXISTS `dedup` AFTER INSERT ON toots FOR EACH ROW BEGIN DELETE FROM toots WHERE rowid NOT IN (SELECT MIN(sortid) FROM toots GROUP BY uri ); END; ")
@ -115,7 +118,7 @@ if not found:
c.execute("CREATE TABLE `toots_temp` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)") c.execute("CREATE TABLE `toots_temp` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)")
for f in following: for f in following:
user_toots = c.execute("SELECT * FROM `toots` WHERE userid LIKE ? ORDER BY id", (f.id,)).fetchall() user_toots = c.execute("SELECT * FROM `toots` WHERE userid LIKE ? ORDER BY id", (f.id,)).fetchall()
if user_toots == None: if user_toots is None:
continue continue
if columns[-1] == "cw": if columns[-1] == "cw":
@ -131,11 +134,13 @@ if not found:
db.commit() db.commit()
def handleCtrlC(signal, frame): def handleCtrlC(signal, frame):
print("\nPREMATURE EVACUATION - Saving chunks") print("\nPREMATURE EVACUATION - Saving chunks")
db.commit() db.commit()
sys.exit(1) sys.exit(1)
signal.signal(signal.SIGINT, handleCtrlC) signal.signal(signal.SIGINT, handleCtrlC)
patterns = { patterns = {
@ -150,7 +155,7 @@ def insert_toot(oii, acc, post, cursor): # extracted to prevent duplication
pid = patterns["pid"].search(oii['object']['id']).group(0) pid = patterns["pid"].search(oii['object']['id']).group(0)
cursor.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", ( cursor.execute("REPLACE INTO toots (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", (
pid, pid,
1 if (oii['object']['summary'] != None and oii['object']['summary'] != "") else 0, 1 if (oii['object']['summary'] is not None and oii['object']['summary'] != "") else 0,
acc.id, acc.id,
oii['object']['id'], oii['object']['id'],
post post
@ -159,16 +164,16 @@ def insert_toot(oii, acc, post, cursor): # extracted to prevent duplication
for f in following: for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY sortid DESC LIMIT 1", (f.id,)).fetchone() last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY sortid DESC LIMIT 1", (f.id,)).fetchone()
if last_toot != None: if last_toot is not None:
last_toot = last_toot[0] last_toot = last_toot[0]
else: else:
last_toot = 0 last_toot = 0
print("Downloading posts for user @{}, starting from {}".format(f.acct, last_toot)) print("Downloading posts for user @{}, starting from {}".format(f.acct, last_toot))
#find the user's activitypub outbox # find the user's activitypub outbox
print("WebFingering...") print("WebFingering...")
instance = patterns["handle"].search(f.acct) instance = patterns["handle"].search(f.acct)
if instance == None: if instance is None:
instance = patterns["url"].search(cfg['site']).group(1) instance = patterns["url"].search(cfg['site']).group(1)
else: else:
instance = instance.group(1) instance = instance.group(1)
@ -182,13 +187,13 @@ for f in following:
r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10) r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10)
# 2. use webfinger to find user's info page # 2. use webfinger to find user's info page
uri = patterns["uri"].search(r.text).group(1) uri = patterns["uri"].search(r.text).group(1)
uri = uri.format(uri = "{}@{}".format(f.username, instance)) uri = uri.format(uri="{}@{}".format(f.username, instance))
r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10) r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10)
j = r.json() j = r.json()
found = False found = False
for link in j['links']: for link in j['links']:
if link['rel'] == 'self': if link['rel'] == 'self':
#this is a link formatted like "https://instan.ce/users/username", which is what we need # this is a link formatted like "https://instan.ce/users/username", which is what we need
uri = link['href'] uri = link['href']
found = True found = True
break break
@ -227,7 +232,7 @@ for f in following:
while not done and len(j['orderedItems']) > 0: while not done and len(j['orderedItems']) > 0:
for oi in j['orderedItems']: for oi in j['orderedItems']:
if oi['type'] != "Create": if oi['type'] != "Create":
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore continue # this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
# its a toost baby # its a toost baby
content = oi['object']['content'] content = oi['object']['content']
@ -236,8 +241,8 @@ for f in following:
try: try:
if pleroma: if pleroma:
if c.execute("SELECT COUNT(*) FROM toots WHERE uri LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0: if c.execute("SELECT COUNT(*) FROM toots WHERE uri LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0:
#we've caught up to the notices we've already downloaded, so we can stop now # we've caught up to the notices we've already downloaded, so we can stop now
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "i know but i don't know how to fix it" # you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "i know but i don't know how to fix it"
done = True done = True
continue continue
if 'lang' in cfg: if 'lang' in cfg:
@ -245,13 +250,13 @@ for f in following:
if oi['object']['contentMap'][cfg['lang']]: # filter for language if oi['object']['contentMap'][cfg['lang']]: # filter for language
insert_toot(oi, f, toot, c) insert_toot(oi, f, toot, c)
except KeyError: except KeyError:
#JSON doesn't have contentMap, just insert the toot irregardlessly # JSON doesn't have contentMap, just insert the toot irregardlessly
insert_toot(oi, f, toot, c) insert_toot(oi, f, toot, c)
else: else:
insert_toot(oi, f, toot, c) insert_toot(oi, f, toot, c)
pass pass
except: except:
pass #ignore any toots that don't successfully go into the DB pass # ignore any toots that don't successfully go into the DB
# get the next/previous page # get the next/previous page
try: try:
@ -285,6 +290,6 @@ for f in following:
print("Done!") print("Done!")
db.commit() db.commit()
db.execute("VACUUM") #compact db db.execute("VACUUM") # compact db
db.commit() db.commit()
db.close() db.close()

View file

@ -4,12 +4,12 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/. # file, You can obtain one at http://mozilla.org/MPL/2.0/.
import mastodon import mastodon
import random, re, json, argparse import re, json, argparse
import functions import functions
from bs4 import BeautifulSoup
parser = argparse.ArgumentParser(description='Reply service. Leave running in the background.') parser = argparse.ArgumentParser(description='Reply service. Leave running in the background.')
parser.add_argument('-c', '--cfg', dest='cfg', default='config.json', nargs='?', parser.add_argument(
'-c', '--cfg', dest='cfg', default='config.json', nargs='?',
help="Specify a custom location for config.json.") help="Specify a custom location for config.json.")
args = parser.parse_args() args = parser.parse_args()
@ -22,16 +22,18 @@ client = mastodon.Mastodon(
access_token=cfg['secret'], access_token=cfg['secret'],
api_base_url=cfg['site']) api_base_url=cfg['site'])
def extract_toot(toot): def extract_toot(toot):
text = functions.extract_toot(toot) text = functions.extract_toot(toot)
text = re.sub(r"^@[^@]+@[^ ]+\s*", r"", text) #remove the initial mention text = re.sub(r"^@[^@]+@[^ ]+\s*", r"", text) # remove the initial mention
text = text.lower() #treat text as lowercase for easier keyword matching (if this bot uses it) text = text.lower() # treat text as lowercase for easier keyword matching (if this bot uses it)
return text return text
class ReplyListener(mastodon.StreamListener): class ReplyListener(mastodon.StreamListener):
def on_notification(self, notification): #listen for notifications def on_notification(self, notification): # listen for notifications
if notification['type'] == 'mention': #if we're mentioned: if notification['type'] == 'mention': # if we're mentioned:
acct = "@" + notification['account']['acct'] #get the account's @ acct = "@" + notification['account']['acct'] # get the account's @
post_id = notification['status']['id'] post_id = notification['status']['id']
# check if we've already been participating in this thread # check if we've already been participating in this thread
@ -44,7 +46,7 @@ class ReplyListener(mastodon.StreamListener):
posts = 0 posts = 0
for post in context['ancestors']: for post in context['ancestors']:
if post['account']['id'] == me: if post['account']['id'] == me:
pin = post["id"] #Only used if pin is called, but easier to call here pin = post["id"] # Only used if pin is called, but easier to call here
posts += 1 posts += 1
if posts >= cfg['max_thread_length']: if posts >= cfg['max_thread_length']:
# stop replying # stop replying
@ -52,12 +54,12 @@ class ReplyListener(mastodon.StreamListener):
return return
mention = extract_toot(notification['status']['content']) mention = extract_toot(notification['status']['content'])
if (mention == "pin") or (mention == "unpin"): #check for keywords if (mention == "pin") or (mention == "unpin"): # check for keywords
print("Found pin/unpin") print("Found pin/unpin")
#get a list of people the bot is following # get a list of people the bot is following
validusers = client.account_following(me) validusers = client.account_following(me)
for user in validusers: for user in validusers:
if user["id"] == notification["account"]["id"]: #user is #valid if user["id"] == notification["account"]["id"]: # user is #valid
print("User is valid") print("User is valid")
visibility = notification['status']['visibility'] visibility = notification['status']['visibility']
if visibility == "public": if visibility == "public":
@ -65,22 +67,23 @@ class ReplyListener(mastodon.StreamListener):
if mention == "pin": if mention == "pin":
print("pin received, pinning") print("pin received, pinning")
client.status_pin(pin) client.status_pin(pin)
client.status_post("Toot pinned!", post_id, visibility=visibility, spoiler_text = cfg['cw']) client.status_post("Toot pinned!", post_id, visibility=visibility, spoiler_text=cfg['cw'])
else: else:
print("unpin received, unpinning") print("unpin received, unpinning")
client.status_post("Toot unpinned!", post_id, visibility=visibility, spoiler_text = cfg['cw']) client.status_post("Toot unpinned!", post_id, visibility=visibility, spoiler_text=cfg['cw'])
client.status_unpin(pin) client.status_unpin(pin)
else: else:
print("User is not valid") print("User is not valid")
else: else:
toot = functions.make_toot(cfg) #generate a toot toot = functions.make_toot(cfg) # generate a toot
toot = acct + " " + toot #prepend the @ toot = acct + " " + toot # prepend the @
print(acct + " says " + mention) #logging print(acct + " says " + mention) # logging
visibility = notification['status']['visibility'] visibility = notification['status']['visibility']
if visibility == "public": if visibility == "public":
visibility = "unlisted" visibility = "unlisted"
client.status_post(toot, post_id, visibility=visibility, spoiler_text = cfg['cw']) #send toost client.status_post(toot, post_id, visibility=visibility, spoiler_text=cfg['cw']) # send toost
print("replied with " + toot) #logging print("replied with " + toot) # logging
rl = ReplyListener() rl = ReplyListener()
client.stream_user(rl) #go! client.stream_user(rl) # go!