diff options
| -rw-r--r-- | .golangci.yml | 1 | ||||
| -rw-r--r-- | README.md | 89 | ||||
| -rw-r--r-- | go.mod | 7 | ||||
| -rw-r--r-- | go.sum | 21 | ||||
| -rw-r--r-- | main.go | 74 | ||||
| -rw-r--r-- | plugins.go | 1 | ||||
| -rw-r--r-- | rss.go | 179 | ||||
| -rw-r--r-- | types.go | 58 |
8 files changed, 398 insertions, 32 deletions
diff --git a/.golangci.yml b/.golangci.yml index 220bddb..7e3e303 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,3 +27,4 @@ linters-settings: - github.com/yuin/gluare - gitlab.com/megalithic-llc/gluasocket - github.com/layeh/gopher-json + - github.com/mmcdole/gofeed @@ -175,23 +175,27 @@ ircChannels = ["#channel1", "#channel2"] Please note that the bot does not have to join a channel to be usable. One can simply query the bot directly as well.<br/> -### databaseUser +#### databaseUser Name of the database user. -### databasePassword +#### databasePassword Password for the database user. -### databaseAddress +#### databaseAddress Address of the database. -### databaseName +#### databaseName Name of the database. -### ircProxy +#### scrapeChannels + +List of channels that the bot will scrape into a database table. You can later on use these databases for the custom commands.<br/> + +#### ircProxy Determines which proxy to use to connect to the IRC network: @@ -199,7 +203,7 @@ Determines which proxy to use to connect to the IRC network: ircProxy = "socks5://127.0.0.1:9050" ``` -### llmProxy +#### llmProxy Determines which proxy to use to connect to the LLM endpoint: @@ -207,30 +211,38 @@ Determines which proxy to use to connect to the LLM endpoint: llmProxy = "socks5://127.0.0.1:9050" ``` -### ircdName +#### ircdName Name of the milla instance, must be unique across all instances. -### adminOnly +#### adminOnly Milla will only answer if the nick is in the admin list. -### webIRCGateway +#### webIRCGateway webirc gateway to use. -### webIRCHostname +#### webIRCHostname webirc hostname to use. -### webIRCPassword +#### webIRCPassword webirc password to use. -### webIRCAddress +#### webIRCAddress webirc address to use. +#### rssFile + +The file that contains the rss feeeds. + +#### channel + +The channel to send the rss feeds to. + ### plugins A list of plugins to load:`plugins = ["./plugins/rss.lua", "./plugins/test.lua"]` @@ -277,6 +289,32 @@ fgColor = 0 bgColor = 28 ``` +## RSS + +The rss file is self-explanatory. Here's an example: + +```json +{ + "feeds": [ + { + "name": "one", + "url": "http://feeds.feedburner.com/crunchyroll/rss", + "proxy": "socks5://172.17.0.1:9007", + "userAgent": "Mozilla/5.0 (X11; U; Linux i686; pl; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1 (Ubuntu-edgy)", + "timeout": 10 + }, + { + "name": "two", + "url": "http://feeds.feedburner.com/crunchyroll/rss/anime", + "proxy": "socks5://172.17.0.1:9007", + "userAgent": "Mozilla/5.0 (X11; U; Linux i686; pl; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1 (Ubuntu-edgy)", + "timeout": 10 + } + ], + "period": 3600 +} +``` + ### Example Config File ```toml @@ -311,10 +349,24 @@ skipTLSVerify = false useTLS = true plugins = ["/plugins/plugin1.lua", "/plugins/plugin2.lua"] adminOnly = false +plugins = ["/plugins/ip.lua", "/plugins/urban.lua", "/plugins/remind.lua"] +[ircd.devinet.watchlist.security] +watchList = ["#securityfeeds"] +watchFiles = ["/watchfiles/voidbox.list"] +alertChannel = "#milla_alerts" +[ircd.devinet.rss.manga] +rssFile = "/rssfeeds/manga.json" +channel = "#manga" +[ircd.devinet.rss.anime] +rssFile = "/rssfeeds/anime.json" +channel = "#anime" [ircd.devinet.watchlist.security] watchList = ["#securityfeeds"] watchFiles = ["/watchfiles/voidbox.list"] alertChannel = "#milla_alerts" +eventTypes = ["PRIVMSG"] +fgColor = 0 +bgColor = 28 [ircd.liberanet] ircServer = "irc.libera.chat" @@ -343,12 +395,12 @@ out = true ircProxy = "socks5://127.0.0.1:9051" llmProxy = "http://127.0.0.1:8181" adminOnly = true -[ircd.devinet_terra.customCommands.digest] +[ircd.liberanet.customCommands.digest] sql = "select log from liberanet_milla_us_market_news order by log desc;" limit = 300 context = ["you are a sentiment-analysis bot"] prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please provide the digest of the news for me." -[ircd.devinet_terra.customCommands.summarize] +[ircd.liberanet.customCommands.summarize] sql= "select log from liberanet_milla_us_market_news order by log desc;" limit= 300 context = ["you are a sentiment-analysis bot"] @@ -393,6 +445,14 @@ Load a plugin: `/load /plugins/rss.lua` Unload a plugin: `/unload /plugins/rss.lua` +#### remind + +Pings the user after the given amount in seconds: `/remind 1200` + +#### roll + +Rolls a number between 1 and 6 if no arguments are given. With one argument it rolls a number between 1 and the given number. With two arguments it rolls a number between the two numbers: `/rool 10000 66666` + ## Deploy ### Docker @@ -768,6 +828,7 @@ isp: Cloudflare, Inc -- query: 1.1.1.1 -- status: success -- regionName: Queensl - Each lua plugin gets its own lua state and will run in a goroutine.<br/> - Lua plugins will not go through a proxy if they are not instructed to do so. If you are using the provided http module, you can set the proxy value before loading the http module as provided in the examples under `plugins`. The module will read and set the following environment variables in the order given: + - `ALL_PROXY` - `HTTPS_PROXY` - `HTTP_PROXY` @@ -12,6 +12,7 @@ require ( github.com/kohkimakimoto/gluayaml v0.0.0-20160815032708-6fe413d49d73 github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e + github.com/mmcdole/gofeed v1.3.0 github.com/sashabaranov/go-openai v1.19.3 github.com/yuin/gluare v0.0.0-20170607022532-d7c94f1a80ed github.com/yuin/gopher-lua v1.1.1 @@ -27,6 +28,8 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/longrunning v0.5.6 // indirect + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -40,7 +43,11 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kr/text v0.2.0 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect @@ -13,6 +13,8 @@ cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXm cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/ailncode/gluaxmlpath v0.0.0-20161126153117-6ce478ecb4a6 h1:FM0WudTZ+xeiXPJcs+X1Zg8JXe4vlb9P2GZYqCYjZkk= github.com/ailncode/gluaxmlpath v0.0.0-20161126153117-6ce478ecb4a6/go.mod h1:Ti1AvV2KUYtHEBX7eYbdAGEfFyKz9+lHrJPcr79Vkng= github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink= @@ -21,6 +23,8 @@ github.com/alecthomas/chroma/v2 v2.12.0 h1:Wh8qLEgMMsN7mgyG8/qIpegky2Hvzr4By6gEF github.com/alecthomas/chroma/v2 v2.12.0/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF0EpseFXdKMBw= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 h1:rdWOzitWlNYeUsXmz+IQfa9NkGEq3gA/qQ3mOEqBU6o= github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9/go.mod h1:X97UjDTXp+7bayQSFZk2hPvCTmTZIicUjZQRtkwgAKY= @@ -69,6 +73,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -88,6 +93,8 @@ github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kohkimakimoto/gluayaml v0.0.0-20160815032708-6fe413d49d73 h1:e2UU76WBjv6W0QUuPHe2QfSAXLR1kouek1fcSUtnSrk= github.com/kohkimakimoto/gluayaml v0.0.0-20160815032708-6fe413d49d73/go.mod h1:I+YgUp/uc0hF+H4RuQjR+uzDJNjUz7Sm21e63UCLWWQ= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -98,6 +105,15 @@ github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:bg6J/5S/AeTz7 github.com/layeh/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:E/q28EyUVBgBQnONAVPIdwvEsv4Ve0vaCA9JWim4+3I= github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e h1:Y86mAFtJjS4P0atZ6QAKH88TV0ASQYJdIGWiOmJKoNY= github.com/lrstanley/girc v0.0.0-20240125042120-9add3166e52e/go.mod h1:lgrnhcF8bg/Bd5HA5DOb4Z+uGqUqGnp4skr+J2GwVgI= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -147,6 +163,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -161,10 +178,14 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -10,6 +10,7 @@ import ( "fmt" "index/suffixarray" "log" + "math/rand" "net" "net/http" "net/url" @@ -222,6 +223,8 @@ func getHelpString() string { helpString += "memstats - returns the memory status currently being used\n" helpString += "load - loads a lua script\n" helpString += "unload - unloads a lua script\n" + helpString += "remind - reminds you in a given amount of seconds\n" + helpString += "roll - rolls a dice. the number is between 1 and 6. One arg sets the upper limit. Two args sets the lower and upper limit in that order\n" return helpString } @@ -546,6 +549,71 @@ func runCommand( } appConfig.deleteLstate(args[1]) + case "remind": + if len(args) < 2 { + client.Cmd.Reply(event, errNotEnoughArgs.Error()) + + break + } + + seconds, err := strconv.Atoi(args[1]) + if err != nil { + client.Cmd.Reply(event, errNotEnoughArgs.Error()) + + break + } + + client.Cmd.Reply(event, "Ok, I'll remind you in "+args[1]+" seconds.") + time.Sleep(time.Duration(seconds) * time.Second) + + client.Cmd.ReplyTo(event, " Ping!") + case "forget": + + client.Cmd.Reply(event, "I no longer even know whether you're supposed to wear or drink a camel.'") + case "roll": + lowerLimit := 1 + upperLimit := 6 + + if len(args) == 1 { + } else if len(args) == 2 { + argOne, err := strconv.Atoi(args[1]) + + if err != nil { + client.Cmd.Reply(event, errNotEnoughArgs.Error()) + + break + } + + upperLimit = argOne + } else if len(args) == 3 { + argOne, err := strconv.Atoi(args[1]) + + if err != nil { + client.Cmd.Reply(event, errNotEnoughArgs.Error()) + + break + } + + lowerLimit = argOne + + argTwo, err := strconv.Atoi(args[2]) + + if err != nil { + client.Cmd.Reply(event, errNotEnoughArgs.Error()) + + break + } + + upperLimit = argTwo + } else { + client.Cmd.Reply(event, errors.New("too many args").Error()) + + break + } + + randomNumber := lowerLimit + rand.Intn(upperLimit-lowerLimit+1) + + client.Cmd.ReplyTo(event, fmt.Sprint(randomNumber)) default: _, ok := appConfig.LuaCommands[args[0]] if !ok { @@ -1262,6 +1330,12 @@ func runIRC(appConfig TomlConfig) { go WatchListHandler(irc, appConfig) } + if len(appConfig.Rss) > 0 { + irc.Handlers.AddBg(girc.CONNECTED, func(client *girc.Client, _ girc.Event) { + go runRSS(&appConfig, irc) + }) + } + for { var dialer proxy.Dialer @@ -343,6 +343,7 @@ func millaModuleLoaderClosure(luaState *lua.LState, client *girc.Client, appConf registerStructAsLuaMetaTable[TomlConfig](luaState, millaModule, checkStruct, TomlConfig{}, "toml_config") registerStructAsLuaMetaTable[CustomCommand](luaState, millaModule, checkStruct, CustomCommand{}, "custom_command") registerStructAsLuaMetaTable[LogModel](luaState, millaModule, checkStruct, LogModel{}, "log_model") + registerStructAsLuaMetaTable[girc.Event](luaState, millaModule, checkStruct, girc.Event{}, "girc_event") luaState.SetGlobal("milla", millaModule) @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "net/url" + "os" + "slices" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/lrstanley/girc" + "github.com/mmcdole/gofeed" + "golang.org/x/net/proxy" +) + +func GetFeed(feed FeedConfig, + feedChanel chan<- *gofeed.Feed, + client *girc.Client, + pool *pgxpool.Pool, + channel, groupName string, +) { + rowName := groupName + "__" + feed.Name + "__" + + parsedFeed, err := feed.FeedParser.ParseURL(feed.URL) + if err != nil { + log.Print(err) + } else { + query := fmt.Sprintf("select newest_unix_time from rss where name = '%s'", rowName) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancel() + + newestFromDB := int64(0) + + err := pool.QueryRow(ctx, query).Scan(&newestFromDB) + if err != nil { + pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName)) + } + + log.Print("Newset from DB: ", newestFromDB) + + sortFunc := func(a, b *gofeed.Item) int { + if a.PublishedParsed.Before(*b.PublishedParsed) { + return -1 + } else if a.PublishedParsed.After(*b.PublishedParsed) { + return 1 + } + + return 0 + } + + slices.SortFunc(parsedFeed.Items, sortFunc) + + for _, item := range parsedFeed.Items { + if item.PublishedParsed.Unix() >= newestFromDB { + client.Cmd.Message(channel, parsedFeed.Title+": "+item.Title) + } + } + + log.Print(parsedFeed.Items[0].PublishedParsed.Unix()) + log.Print(parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix()) + + query = fmt.Sprintf("update rss set newest_unix_time = %d where name = '%s'", parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix(), rowName) + + ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancel() + + _, err = pool.Exec(ctx2, query) + if err != nil { + log.Print(err) + } + } + + feedChanel <- parsedFeed +} + +func feedDispatcher( + config RSSConfig, + client *girc.Client, + pool *pgxpool.Pool, + channel, groupName string, +) { + feedChanel := make(chan *gofeed.Feed) + + for i := range len(config.Feeds) { + config.Feeds[i].FeedParser = gofeed.NewParser() + + config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent + + if config.Feeds[i].Proxy != "" { + proxyURL, err := url.Parse(config.Feeds[i].Proxy) + if err != nil { + log.Print(err) + continue + } + + dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second}) + if err != nil { + log.Print(err) + + continue + } + + httpClient := http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + }, + } + + config.Feeds[i].FeedParser.Client = &httpClient + } + } + + for _, feed := range config.Feeds { + go GetFeed(feed, feedChanel, client, pool, channel, groupName) + } + + // <-feedChanel +} + +func ParseRSSConfig(rssConfFilePath string) *RSSConfig { + file, err := os.Open(rssConfFilePath) + if err != nil { + log.Print(err) + + return nil + } + + var config *RSSConfig + + decoder := json.NewDecoder(file) + + err = decoder.Decode(&config) + if err != nil { + log.Print(err) + + return nil + } + + return config +} + +func runRSS(appConfig *TomlConfig, client *girc.Client) { + for { + query := fmt.Sprintf( + `create table if not exists rss ( + id serial primary key, + name text not null unique, + newest_unix_time bigint not null + )`) + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.RequestTimeout)*time.Second) + defer cancel() + + _, err := appConfig.pool.Exec(ctx, query) + if err != nil { + log.Print(err) + time.Sleep(time.Duration(appConfig.MillaReconnectDelay) * time.Second) + } else { + for groupName, rss := range appConfig.Rss { + log.Print("RSS: joining ", rss.Channel) + client.Cmd.Join(rss.Channel) + rssConfig := ParseRSSConfig(rss.RssFile) + if rssConfig == nil { + log.Print("Could not parse RSS config file " + rss.RssFile + ". Exiting.") + } else { + for { + feedDispatcher(*rssConfig, client, appConfig.pool, rss.Channel, groupName) + time.Sleep(time.Duration(rssConfig.Period) * time.Second) + } + } + } + } + } +} @@ -5,6 +5,7 @@ import ( "time" "github.com/jackc/pgx/v5/pgxpool" + "github.com/mmcdole/gofeed" lua "github.com/yuin/gopher-lua" ) @@ -43,6 +44,11 @@ type LuaCommand struct { FuncName string } +type RssFile struct { + RssFile string `toml:"rssFile"` + Channel string `toml:"channel"` +} + type TomlConfig struct { IrcServer string `toml:"ircServer"` IrcNick string `toml:"ircNick"` @@ -70,29 +76,31 @@ type TomlConfig struct { WebIRCGateway string `toml:"webIRCGateway"` WebIRCHostname string `toml:"webIRCHostname"` WebIRCAddress string `toml:"webIRCAddress"` + RSSFile string `toml:"rssFile"` Plugins []string `toml:"plugins"` CustomCommands map[string]CustomCommand `toml:"customCommands"` WatchLists map[string]WatchList `toml:"watchList"` LuaStates map[string]LuaLstates LuaCommands map[string]LuaCommand - Temp float64 `toml:"temp"` - RequestTimeout int `toml:"requestTimeout"` - MillaReconnectDelay int `toml:"millaReconnectDelay"` - IrcPort int `toml:"ircPort"` - KeepAlive int `toml:"keepAlive"` - MemoryLimit int `toml:"memoryLimit"` - PingDelay int `toml:"pingDelay"` - PingTimeout int `toml:"pingTimeout"` - TopP float32 `toml:"topP"` - TopK int32 `toml:"topK"` - EnableSasl bool `toml:"enableSasl"` - SkipTLSVerify bool `toml:"skipTLSVerify"` - UseTLS bool `toml:"useTLS"` - DisableSTSFallback bool `toml:"disableSTSFallback"` - AllowFlood bool `toml:"allowFlood"` - Debug bool `toml:"debug"` - Out bool `toml:"out"` - AdminOnly bool `toml:"adminOnly"` + Rss map[string]RssFile `toml:"rss"` + Temp float64 `toml:"temp"` + RequestTimeout int `toml:"requestTimeout"` + MillaReconnectDelay int `toml:"millaReconnectDelay"` + IrcPort int `toml:"ircPort"` + KeepAlive int `toml:"keepAlive"` + MemoryLimit int `toml:"memoryLimit"` + PingDelay int `toml:"pingDelay"` + PingTimeout int `toml:"pingTimeout"` + TopP float32 `toml:"topP"` + TopK int32 `toml:"topK"` + EnableSasl bool `toml:"enableSasl"` + SkipTLSVerify bool `toml:"skipTLSVerify"` + UseTLS bool `toml:"useTLS"` + DisableSTSFallback bool `toml:"disableSTSFallback"` + AllowFlood bool `toml:"allowFlood"` + Debug bool `toml:"debug"` + Out bool `toml:"out"` + AdminOnly bool `toml:"adminOnly"` pool *pgxpool.Pool Admins []string `toml:"admins"` IrcChannels []string `toml:"ircChannels"` @@ -169,3 +177,17 @@ type MemoryElement struct { Role string `json:"role"` Content string `json:"content"` } + +type FeedConfig struct { + Name string `json:"name"` + URL string `json:"url"` + UserAgent string `json:"userAgent"` + Proxy string `json:"proxy"` + Timeout int `json:"timeout"` + FeedParser *gofeed.Parser +} + +type RSSConfig struct { + Feeds []FeedConfig `json:"feeds"` + Period int `json:"period"` +} |