diff options
| -rw-r--r-- | .golangci.yml | 1 | ||||
| -rw-r--r-- | Dockerfile | 4 | ||||
| -rw-r--r-- | Dockerfile_debug | 14 | ||||
| -rw-r--r-- | Dockerfile_distroless | 2 | ||||
| -rw-r--r-- | Dockerfile_distroless_vendored | 2 | ||||
| -rw-r--r-- | README.md | 413 | ||||
| -rw-r--r-- | go.mod | 45 | ||||
| -rw-r--r-- | go.sum | 86 | ||||
| -rw-r--r-- | main.go | 298 | ||||
| -rw-r--r-- | openrouter.go | 1 | ||||
| -rw-r--r-- | plugins.go | 138 | ||||
| -rw-r--r-- | plugins/urban.lua | 3 | ||||
| -rw-r--r-- | rss.go | 91 | ||||
| -rw-r--r-- | types.go | 63 |
14 files changed, 635 insertions, 526 deletions
diff --git a/.golangci.yml b/.golangci.yml index 7e3e303..8575a33 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -28,3 +28,4 @@ linters-settings: - gitlab.com/megalithic-llc/gluasocket - github.com/layeh/gopher-json - github.com/mmcdole/gofeed + - github.com/cenkalti/backoff/v5 @@ -1,11 +1,11 @@ -FROM golang:1.22-alpine3.20 AS builder +FROM golang:1.23-alpine3.21 AS builder WORKDIR /milla COPY go.sum go.mod /milla/ RUN go mod download COPY *.go /milla/ RUN go build -FROM alpine:3.20 +FROM alpine:3.21 ENV HOME /home/user RUN set -eux; \ adduser -u 1001 -D -h "$HOME" user; \ diff --git a/Dockerfile_debug b/Dockerfile_debug new file mode 100644 index 0000000..ee2ec57 --- /dev/null +++ b/Dockerfile_debug @@ -0,0 +1,14 @@ +FROM golang:1.23-alpine3.21 AS builder +WORKDIR /milla +COPY go.sum go.mod /milla/ +COPY vendor /milla/vendor +COPY *.go /milla/ +RUN CGO_ENABLED=0 go build + +FROM golang:1.23-alpine3.21 AS debug +RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest + +FROM alpine:3.21 +COPY --from=debug /go/bin/dlv /usr/bin/dlv +COPY --from=builder /milla/milla "/usr/bin/milla" +ENTRYPOINT ["/usr/bin/dlv"] diff --git a/Dockerfile_distroless b/Dockerfile_distroless index a5a7c47..275d301 100644 --- a/Dockerfile_distroless +++ b/Dockerfile_distroless @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine3.20 AS builder +FROM golang:1.23-alpine3.21 AS builder WORKDIR /milla COPY go.sum go.mod /milla/ RUN go mod download diff --git a/Dockerfile_distroless_vendored b/Dockerfile_distroless_vendored index 8d8f3ab..0467cba 100644 --- a/Dockerfile_distroless_vendored +++ b/Dockerfile_distroless_vendored @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine3.20 AS builder +FROM golang:1.23-alpine3.21 AS builder WORKDIR /milla COPY go.sum go.mod /milla/ COPY vendor /milla/vendor @@ -25,259 +25,68 @@ The bot will see a chat prompt as a command if the message begins with `botnick: An example is provided under `config-example.toml`. Please note that all the config options are specific to one instance which is defined by `ircd.nameofyourinstance`.<br/> -#### ircServer - -The address for the IRC server to connect to. - -#### ircNick - -The nick the bot should use. - -#### enableSasl - -Whether to use SASL for authentication. - -#### ircSaslUser - -The SASL username. - -#### ircSaslPass - -The SASL password for SASL plain authentication. Can also be passed as and environment variable. - -#### Endpoint - -The address for the Ollama chat endpoint. - -#### model - -The name of the model to use. - -#### chromaStyle - -The style to use for syntax highlighting done by [chroma](https://github.com/alecthomas/chroma). This is basically what's called a "theme". - -#### chromaFormatter - -The formatter to use. This tells chroma how to generate the color in the output. The supported options are: - -- `noop` for no syntax highlighting -- `terminal` for 8-color terminals -- `terminal8` for 8-color terminals -- `terminal16` for 16-color terminals -- `terminal256` for 256-color terminals -- `terminal16m` for truecolor terminals -- `html` for HTML output - -**_NOTE_**: please note that the terminal formatters will increase the size of the IRC event. Depending on the IRC server, this may or may not be a problem. - -#### provider - -Which LLM provider to use. The supported options are: - -- [ollama](https://github.com/ollama/ollama) -- chatgpt -- gemini -- [openrouter](https://openrouter.ai/) - -#### apikey - -The apikey to use for the LLM provider. Can also be passed as and environment variable. - -#### ollamaSystem - -The system message to use for ollama. - -#### clientCertPath - -The path to the client certificate to use for client cert authentication. - -#### serverPass - -The password to use for the IRC server the bot is trying to connect to if the server has a password. Can also be passed as and environment variable. - -#### bind - -Which address to bind to for the IRC server. - -#### temp - -The temperature to config the model with. - -#### requestTimeout - -The timeout for requests made to the LLM provider. - -#### millaReconnectDelay - -How much to wait before reconnecting to the IRC server. - -#### ircPort - -Which port to connect to for the IRC server. - -#### keepAlive - -#### memoryLimit - -How many conversations to keep in memory for a model. - -#### pingDelay - -Ping delay for the IRC server. - -#### pingTimeout - -Ping timeout for the IRC server. - -#### topP - -set the Top_p paramater - -#### topK - -set the Top_k parameter - -#### skipTLSVerify - -Skip verifying the IRC server's TLS certificate. This only makes sense if you are trying to connect to an IRC server with a self-signed certificate. - -#### useTLS - -Whether to use TLS to connect to the IRC server. This option is provided to support usage on overlay networks such as Tor, i2p and [yggdrassil](https://github.com/yggdrasil-network/yggdrasil-go). - -#### disableSTSFallback - -Disables the "fallback" to a non-TLS connection if the strict transport policy expires and the first attempt to reconnect back to the TLS version fails. - -#### allowFlood - -Disable [girc](https://github.com/lrstanley/girc)'s built-in flood protection. - -#### debug - -Whether to enable debug logging. The logs are written to stdout. - -#### out - -Whether to write raw messages to stdout. - -#### admins - -List of admins for the bot. Only admins can use commands. - -``` -admins = ["admin1", "admin2"] -``` - -#### ircChannels - -List of channels for the bot to join when it connects to the server. - -``` -ircChannels = [["#channel1","channel1password"], ["#channel2",""], ["#channel3"]] -``` - -In the provided example, milla will attempt to join `#channel1` with the provided password while for the other two channels, it will try to join normally.<br/> -This behaviour is consistant across all places where a channel name is the input.<br/> - -Please note that the bot does not have to join a channel to be usable. One can simply query the bot directly as well.<br/> - -#### databaseUser - -Name of the database user. - -#### databasePassword - -Password for the database user. - -#### databaseAddress - -Address of the database. - -#### databaseName - -Name of the database. - -#### scrapeChannels - -List of channels that the bot will scrape into a database table. You can later on use these databases for the custom commands.<br/> - -``` -ircChannels = [["#channel1","channel1password"], ["#channel2",""], ["#channel3"]] -``` - -#### ircProxy - -Determines which proxy to use to connect to the IRC network: - -``` -ircProxy = "socks5://127.0.0.1:9050" -``` - -#### llmProxy - -Determines which proxy to use to connect to the LLM endpoint: - -``` -llmProxy = "socks5://127.0.0.1:9050" -``` - -#### generalProxy - -Determines which proxy to use for other things: - -``` -llmProxy = "socks5://127.0.0.1:9050" -``` - -#### ircdName - -Name of the milla instance, must be unique across all instances. - -#### adminOnly - -Milla will only answer if the nick is in the admin list. - -#### webIRCGateway - -webirc gateway to use. - -#### webIRCHostname - -webirc hostname to use. - -#### webIRCPassword - -webirc password to use. - -#### webIRCAddress - -webirc address to use. - -#### context - -the context to use for the normal conversations with the bot. Yes, this is how you tell your milla instance to act like a pirate. - -```toml -context = ["you are a pirate. use the language and words a pirate would unless you are asked to do otherwise explicitly", "your name is caption blackbeard"] -``` - -```toml -context = ["please respond in french even if i use another language unless you are specifically asked to use any language other than french", "your name is terra"] -``` - -#### rssFile - -The file that contains the rss feeeds. - -#### channel - -The channel to send the rss feeds to. - -### plugins - -A list of plugins to load:`plugins = ["./plugins/rss.lua", "./plugins/test.lua"]` +| Option | Description | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ircServer | The address for the IRC server to connect to | +| ircNick | The nick the bot should use | +| enableSasl | Whether to use SASL for authentication | +| ircSaslUser | The SASL username | +| ircSaslPass | The SASL password for SASL plain authentication. Can also be passed as and environment variable | +| Endpoint | The address for the Ollama chat endpoint | +| model | The name of the model to use | +| chromaStyle | The style to use for syntax highlighting done by [chroma](https://github.com/alecthomas/chroma). This is basically what's called a "theme" | +| chromaFormatter | The formatter to use. This tells chroma how to generate the color in the output. The supported options are:<br><br>- `noop` for no syntax highlighting<br>- `terminal` for 8-color terminals<br>- `terminal8` for 8-color terminals<br>- `terminal16` for 16-color terminals<br>- `terminal256` for 256-color terminals<br>- `terminal16m` for truecolor terminals<br>- `html` for HTML output<br><br>**_NOTE_**: please note that the terminal formatters will increase the size of the IRC event. Depending on the IRC server, this may or may not be a problem. | +| provider | Which LLM provider to use. The supported options are:<br><br>- [ollama](https://github.com/ollama/ollama)<br>- chatgpt<br>- gemini<br>- [openrouter](https://openrouter.ai/)<br> | +| apikey | The apikey to use for the LLM provider. Can also be passed as and environment variable | +| clientCertPath | The path to the client certificate to use for client cert authentication | +| serverPass | The password to use for the IRC server the bot is trying to connect to if the server has a password. Can also be passed as and environment variable | +| bind | Which address to bind to for the IRC server | +| requestTimeout | The timeout for requests made to the LLM provider | +| millaReconnectDelay | How much to wait before reconnecting to the IRC server | +| ircPort | Which port to connect to for the IRC server | +| keepAlive | | +| memoryLimit | How many conversations to keep in memory for a model | +| pingDelay | Ping delay for the IRC server | +| pingTimeout | Ping timeout for the IRC server | +| skipTLSVerify | Skip verifying the IRC server's TLS certificate. This only makes sense if you are trying to connect to an IRC server with a self-signed certificate | +| useTLS | Whether to use TLS to connect to the IRC server. This option is provided to support usage on overlay networks such as Tor, i2p and [yggdrassil](https://github.com/yggdrasil-network/yggdrasil-go) | +| disableSTSFallback | Disables the "fallback" to a non-TLS connection if the strict transport policy expires and the first attempt to reconnect back to the TLS version fails | +| allowFlood | Disable [girc](https://github.com/lrstanley/girc)'s built-in flood protection | +| debug | Whether to enable debug logging. The logs are written to stdout | +| out | Whether to write raw messages to stdout | +| admins | List of admins for the bot. Only admins can use commands.<br><br>`admins = ["admin1", "admin2"]`<br> | +| ircChannels | List of channels for the bot to join when it connects to the server.<br>`ircChannels = [["#channel1","channel1password"], ["#channel2",""], ["#channel3"]]`<br>In the provided example, milla will attempt to join `#channel1` with the provided password while for the other two channels, it will try to join normally.<br><br>**_NOTE 1_**: This behaviour is consistant across all places where a channel name is the input.<br><br>**_NOTE 2_**: Please note that the bot does not have to join a channel to be usable. One can simply query the bot directly as well.<br> | +| databaseUser | Name of the database user | +| databasePassword | Password for the database user | +| databaseAddress | Address of the database | +| databaseName | Name of the database | +| scrapeChannels | List of channels that the bot will scrape into a database table. You can later on use these databases for the custom commands.<br><br>`ircChannels = [["#channel1","channel1password"], ["#channel2",""], ["#channel3"]]` | +| ircProxy | Determines which proxy to use to connect to the IRC network:<br>`ircProxy = "socks5://127.0.0.1:9050"` | +| llmProxy | Determines which proxy to use to connect to the LLM endpoint:<br>`llmProxy = "socks5://127.0.0.1:9050"` | +| generalProxy | Determines which proxy to use for other things:<br>`llmProxy = "socks5://127.0.0.1:9050"`<br><br>**_NOTE_**: Lua scripts do not use the `generalProxy` option. They will use whatever proxy that the invidividual script has them use. The RSS functionaly lets you use a proxy for every single entry. | +| ircdName | Name of the milla instance, must be unique across all instances | +| adminOnly | Milla will only answer if the nick is in the admin list | +| webIRCGateway | webirc gateway to use | +| webIRCHostname | webirc hostname to use | +| webIRCPassword | webirc password to use | +| webIRCAddress | webirc address to use | +| context | Artificially provide a history of messages for the bot.<br><br>`tomlcontext = ["you are a pirate. use the language and words a pirate would unless you are asked to do otherwise explicitly", "your name is caption blackbeard"]`<br>`tomlcontext = ["please respond in french even if i use another language unless you are specifically asked to use any language other than french", "your name is terra"]` | +| rssFile | The file that contains the rss feeeds | +| channel | The channel to send the rss feeds to | +| plugins | A list of plugins to load:`plugins = ["./plugins/rss.lua", "./plugins/test.lua"]` | +| systemPrompt | The system prompt for the AI chat bot | +| temperature | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| topP | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| topK | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaMirostat | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaMirostatEta | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaMirostatTau | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaNumCtx | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaRepeatLastN | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaRepeatPenalty | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaSeed | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaNumPredict | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | +| ollamaMinp | [ollama docs](https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values) | ## Custom Commands @@ -287,25 +96,29 @@ Custom commands let you define a command that does a SQL query to the database a [ircd.devinet_terra.customCommands.digest] sql = "select log from liberanet_milla_us_market_news order by log desc;" limit = 300 -context = ["you are a sentiment-analysis bot"] +context = ["",""] +systemPrompt = ["you are a sentiment analysis bot."] prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please provide the digest of the news for me." [ircd.devinet_terra.customCommands.summarize] sql= "select log from liberanet_milla_us_market_news order by log desc;" limit= 300 -context = ["you are a sentiment-analysis bot"] +systemPrompt = ["you are a sentiment-analysis bot"] prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please summarize the provided news for me. provide some details." [ircd.devinet_terra.customCommands.canada] sql= "select log from liberanet_milla_us_market_news order by log desc;" limit= 300 -context = ["you are a canadian news anchor", "you only care about news that is relevant to canada"] prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please summarize the provided news for me. provide some details." ``` -In the above example digest and summarize will be the names of the commands: `milla: /cmd summarize`.<br/> -Currently you should only ask for the log column in the query. Asking for the other column will result in the query not succeeding.<br/> -The `limit` parameter limits the number of SQL queries that are used to generate the response. Whether you hit the token limit of the provider you use and the cost is something you should be aware of.<br/> -A `limit` value of 0 disables the limit on the amount of rows that are passed to milla.<br/> -NOTE: since each milla instance can have its own database, all instances might not necessarily have access to all the data milla is gathering. If you use the same database for all the instances, all instances will have access to all the gathered data.<br/> +In the above example digest and summarize will be the names of the commands: `milla: /cmd summarize`. + +Currently you should only ask for the log column in the query. Asking for the other column will result in the query not succeeding. + +The `limit` parameter limits the number of SQL queries that are used to generate the response. Whether you hit the token limit of the provider you use and the cost is something you should be aware of. + +A `limit` value of 0 disables the limit on the amount of rows that are passed to milla. + +**_NOTE_**: since each milla instance can have its own database, all instances might not necessarily have access to all the data milla is gathering. If you use the same database for all the instances, all instances will have access to all the gathered data. ## Watchlist @@ -383,7 +196,7 @@ skipTLSVerify = false useTLS = true adminOnly = false plugins = ["/plugins/ip.lua", "/plugins/urban.lua"] -context = ["please respond in french even if i use another language unless you are specifically asked to use any language other than french"] +systemPrompt = ["please respond in french even if i use another language unless you are specifically asked to use any language other than french"] [ircd.devinet.watchlist.security] watchList = [["#securityfeeds"]] watchFiles = ["/watchfiles/voidbox.list"] @@ -428,65 +241,31 @@ adminOnly = true [ircd.liberanet.customCommands.digest] sql = "select log from liberanet_milla_us_market_news order by log desc;" limit = 300 -context = ["you are a sentiment-analysis bot"] +systemPrompt = ["you are a sentiment-analysis bot"] prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please provide the digest of the news for me." [ircd.liberanet.customCommands.summarize] sql= "select log from liberanet_milla_us_market_news order by log desc;" limit= 300 -context = ["you are a sentiment-analysis bot"] +systemPrompt = ["you are a sentiment-analysis bot"] prompt= "i have provided to you news headlines in the form of previous conversations between you and me using the user role. please summarize the provided news for me. provide some details." ``` ## Commands -#### help - -Prints the help message. - -#### get - -Get the value of a config option. Use the same name as the config file but capitalized: `/get chromaFormatter` - -#### getall - -Get the value of all config options. - -#### set - -Set a config option on the fly. Use the same name as the config file but capitalized: `/set chromaFormatter noop` - -#### memstats - -Returns memory stats for milla. - -#### join - -Joins a channel: `/join #channel [optional_password]` - -#### leave - -Leaves a channel: `/leave #channel` - -#### load - -Load a plugin: `/load /plugins/rss.lua` - -#### unload - -Unload a plugin: `/unload /plugins/rss.lua` - -#### remind - -Pings the user after the given amount in seconds: `/remind 1200` - -#### roll - -Rolls a number between 1 and 6 if no arguments are given. With one argument it rolls a number between 1 and the given number. With two arguments it rolls a number between the two numbers: `/rool 10000 66666` - -#### whois - -IANA whois endpoint query: `milla: /whois xyz` -This command uses the `generalProxy` option. +| Command | Description | +| -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| help | Prints the help message | +| get | Get the value of a config option. Use the same name as the config file but capitalized: `/get chromaFormatter` | +| getall | Get the value of all config options | +| set | Set a config option on the fly. Use the same name as the config file but capitalized: `/set chromaFormatter noop` | +| memstats | Returns memory stats for milla | +| join | Joins a channel: `/join #channel [optional_password]` | +| leave | Leaves a channel: `/leave #channel` | +| load | Load a plugin: `/load /plugins/rss.lua` | +| unload | Unload a plugin: `/unload /plugins/rss.lua` | +| remind | Pings the user after the given amount in seconds: `/remind 1200` | +| roll | Rolls a number between 1 and 6 if no arguments are given. With one argument it rolls a number between 1 and the given number. With two arguments it rolls a number between the two numbers: `/roll 10000 66666` | +| whois | IANA whois endpoint query: `milla: /whois xyz`. This command uses the `generalProxy` option. | ## Deploy @@ -754,19 +533,19 @@ milla.part_channel(channel) ``` ```lua -milla.send_ollama_request(prompt) +milla.send_ollama_request(prompt, systemPrompt) ``` ```lua -milla.send_gemini_request(prompt) +milla.send_gemini_request(prompt, systemPrompt) ``` ```lua -milla.send_chatgpt_request(prompt) +milla.send_chatgpt_request(prompt, systemPrompt) ``` ```lua -milla.send_or_request(prompt) +milla.send_or_request(prompt, systemPrompt) ``` ```lua @@ -849,7 +628,7 @@ ALL_PROXY=socks5://172.17.0.1:9050 More of milla's functionality will be available through milla's lua module over time.<br/> -The following libraries are loaded by milla by default: +milla loads the following libraries by default: - [gluaxmlpath](https://github.com/ailncode/gluaxmlpath) - [gluahttp](https://github.com/cjoudrey/gluahttp) @@ -1,11 +1,14 @@ module milla -go 1.22.3 +go 1.23 + +toolchain go1.23.4 require ( github.com/BurntSushi/toml v0.3.1 github.com/ailncode/gluaxmlpath v0.0.0-20161126153117-6ce478ecb4a6 github.com/alecthomas/chroma/v2 v2.12.0 + github.com/cenkalti/backoff/v5 v5.0.1 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 github.com/google/generative-ai-go v0.11.2 github.com/jackc/pgx/v5 v5.5.5 @@ -17,17 +20,17 @@ require ( github.com/yuin/gluare v0.0.0-20170607022532-d7c94f1a80ed github.com/yuin/gopher-lua v1.1.1 gitlab.com/megalithic-llc/gluasocket v0.3.1 - golang.org/x/net v0.24.0 - google.golang.org/api v0.176.1 + golang.org/x/net v0.26.0 + google.golang.org/api v0.186.0 ) require ( - cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/ai v0.4.0 // indirect - cloud.google.com/go/auth v0.3.0 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/ai v0.8.0 // indirect + cloud.google.com/go/auth v0.6.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect - cloud.google.com/go/longrunning v0.5.6 // indirect + cloud.google.com/go/longrunning v0.5.7 // indirect github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect @@ -39,7 +42,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.4 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect @@ -50,21 +53,21 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/oauth2 v0.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/grpc v1.64.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) @@ -1,16 +1,16 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= -cloud.google.com/go/ai v0.4.0 h1:hoF8+joXKfW2Ug7MKssoffXCVUSxUqMUJL0hJxVtO1Q= -cloud.google.com/go/ai v0.4.0/go.mod h1:iX72tmUodGXVDxRDCGUZEPiB9HaMeERXkOdgCkUi8sA= -cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs= -cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/ai v0.8.0 h1:rXUEz8Wp2OlrM8r1bfmpF2+VKqc1VJpafE3HgzRnD/w= +cloud.google.com/go/ai v0.8.0/go.mod h1:t3Dfk4cM61sytiggo2UyGsDVW3RF1qGZaUKDrZFyqkE= +cloud.google.com/go/auth v0.6.0 h1:5x+d6b5zdezZ7gmLWD1m/xNjnaQ2YDhmIz/HH3doy1g= +cloud.google.com/go/auth v0.6.0/go.mod h1:b4acV+jLQDyjwm4OXHYjNvRi4jvGBzHWJRtJcy+2P4g= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXmjuEaE= -cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= @@ -25,6 +25,8 @@ github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/cenkalti/backoff/v5 v5.0.1 h1:kGZdCHH1+eW+Yd0wftimjMuhg9zidDvNF5aGdnkkb+U= +github.com/cenkalti/backoff/v5 v5.0.1/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 h1:rdWOzitWlNYeUsXmz+IQfa9NkGEq3gA/qQ3mOEqBU6o= github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9/go.mod h1:X97UjDTXp+7bayQSFZk2hPvCTmTZIicUjZQRtkwgAKY= @@ -81,8 +83,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= -github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -129,8 +131,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/gluare v0.0.0-20170607022532-d7c94f1a80ed h1:I1vcLHWU9m30rA90rMrKPu0eD3NDA4FBlkB8WMaDyUw= github.com/yuin/gluare v0.0.0-20170607022532-d7c94f1a80ed/go.mod h1:9w6KSdZh23UWqOywWsRLUcJUrUNjRh4Ql3z9uVgnSP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= @@ -139,20 +141,20 @@ gitlab.com/megalithic-llc/gluasocket v0.3.1 h1:CtsSTZa3G5WnMbhZ3TgvpLwpVlQv6KjO2 gitlab.com/megalithic-llc/gluasocket v0.3.1/go.mod h1:/FBarfbXYDsAPzx16ZUmql2rq7GLU5HeuhaJOC9DeBw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -164,11 +166,11 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= -golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -180,14 +182,14 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -196,24 +198,24 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.176.1 h1:DJSXnV6An+NhJ1J+GWtoF2nHEuqB1VNoTfnIbjNvwD4= -google.golang.org/api v0.176.1/go.mod h1:j2MaSDYcvYV1lkZ1+SMW4IeF90SrEyFA+tluDYWRrFg= +google.golang.org/api v0.186.0 h1:n2OPp+PPXX0Axh4GuSsL5QL8xQCTb2oDwyzPnQvqUug= +google.golang.org/api v0.186.0/go.mod h1:hvRbBmgoje49RV3xqVXrmP6w93n6ehGgIVPYrGtBFFc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -223,8 +225,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -6,6 +6,7 @@ import ( "crypto/tls" "encoding/json" "errors" + "expvar" "flag" "fmt" "index/suffixarray" @@ -13,6 +14,7 @@ import ( "math/rand" "net" "net/http" + _ "net/http/pprof" "net/url" "os" "os/signal" @@ -26,6 +28,7 @@ import ( "github.com/BurntSushi/toml" "github.com/alecthomas/chroma/v2/quick" + "github.com/cenkalti/backoff/v5" "github.com/google/generative-ai-go/genai" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -69,8 +72,8 @@ func addSaneDefaults(config *TomlConfig) { config.DatabaseName = "milladb" } - if config.Temp == 0 { - config.Temp = 0.5 + if config.Temperature == 0 { + config.Temperature = 0.5 } if config.RequestTimeout == 0 { @@ -101,9 +104,49 @@ func addSaneDefaults(config *TomlConfig) { config.PingTimeout = 20 } + if config.OllamaMirostatEta == 0 { + config.OllamaMirostatEta = 0.1 + } + + if config.OllamaMirostatTau == 0 { + config.OllamaMirostatTau = 5.0 + } + + if config.OllamaNumCtx == 0 { + config.OllamaNumCtx = 4096 + } + + if config.OllamaRepeatLastN == 0 { + config.OllamaRepeatLastN = 64 + } + + if config.OllamaRepeatPenalty == 0 { + config.OllamaRepeatPenalty = 1.1 + } + + if config.OllamaSeed == 0 { + config.OllamaSeed = 42 + } + + if config.OllamaNumPredict == 0 { + config.OllamaNumPredict = -1 + } + + if config.TopK == 0 { + config.TopK = 40 + } + if config.TopP == 0.0 { config.TopP = 0.9 } + + if config.OllamaMinP == 0 { + config.OllamaMinP = 0.05 + } + + if config.Temperature == 0 { + config.Temperature = 0.7 + } } func getTableFromChanName(channel, ircdName string) string { @@ -316,7 +359,7 @@ func handleCustomCommand( bigPrompt += log.Log + "\n" } - result := ChatGPTRequestProcessor(appConfig, client, event, &gptMemory, customCommand.Prompt) + result := ChatGPTRequestProcessor(appConfig, client, event, &gptMemory, customCommand.Prompt, customCommand.SystemPrompt) if result != "" { SendToIRC(client, event, result, appConfig.ChromaFormatter) } @@ -341,7 +384,7 @@ func handleCustomCommand( }) } - result := GeminiRequestProcessor(appConfig, client, event, &geminiMemory, customCommand.Prompt) + result := GeminiRequestProcessor(appConfig, client, event, &geminiMemory, customCommand.Prompt, customCommand.SystemPrompt) if result != "" { SendToIRC(client, event, result, appConfig.ChromaFormatter) } @@ -362,7 +405,7 @@ func handleCustomCommand( }) } - result := OllamaRequestProcessor(appConfig, client, event, &ollamaMemory, customCommand.Prompt) + result := OllamaRequestProcessor(appConfig, client, event, &ollamaMemory, customCommand.Prompt, customCommand.SystemPrompt) if result != "" { SendToIRC(client, event, result, appConfig.ChromaFormatter) } @@ -541,7 +584,7 @@ func runCommand( appConfig.deleteLstate(args[1]) case "remind": - if len(args) < 2 { + if len(args) < 2 { //nolint: mnd,gomnd client.Cmd.Reply(event, errNotEnoughArgs.Error()) break @@ -559,10 +602,9 @@ func runCommand( client.Cmd.ReplyTo(event, " Ping!") case "forget": - client.Cmd.Reply(event, "I no longer even know whether you're supposed to wear or drink a camel.'") case "whois": - if len(args) < 2 { + if len(args) < 2 { //nolint: mnd,gomnd client.Cmd.Reply(event, errNotEnoughArgs.Error()) break @@ -575,9 +617,8 @@ func runCommand( upperLimit := 6 if len(args) == 1 { - } else if len(args) == 2 { + } else if len(args) == 2 { //nolint: mnd,gomnd argOne, err := strconv.Atoi(args[1]) - if err != nil { client.Cmd.Reply(event, errNotEnoughArgs.Error()) @@ -585,9 +626,8 @@ func runCommand( } upperLimit = argOne - } else if len(args) == 3 { + } else if len(args) == 3 { //nolint: mnd,gomnd argOne, err := strconv.Atoi(args[1]) - if err != nil { client.Cmd.Reply(event, errNotEnoughArgs.Error()) @@ -597,7 +637,6 @@ func runCommand( lowerLimit = argOne argTwo, err := strconv.Atoi(args[2]) - if err != nil { client.Cmd.Reply(event, errNotEnoughArgs.Error()) @@ -636,7 +675,7 @@ func runCommand( func DoOllamaRequest( appConfig *TomlConfig, ollamaMemory *[]MemoryElement, - prompt string, + prompt, systemPrompt string, ) (string, error) { var jsonPayload []byte @@ -665,14 +704,25 @@ func DoOllamaRequest( KeepAlive: time.Duration(appConfig.KeepAlive), Stream: false, Messages: *ollamaMemory, + System: systemPrompt, Options: OllamaRequestOptions{ - Temperature: appConfig.Temp, + Mirostat: appConfig.OllamaMirostat, + MirostatEta: appConfig.OllamaMirostatEta, + MirostatTau: appConfig.OllamaMirostatTau, + NumCtx: appConfig.OllamaNumCtx, + RepeatLastN: appConfig.OllamaRepeatLastN, + RepeatPenalty: appConfig.OllamaRepeatPenalty, + Temperature: appConfig.Temperature, + Seed: appConfig.OllamaSeed, + NumPredict: appConfig.OllamaNumPredict, + TopK: appConfig.TopK, + TopP: appConfig.TopP, + MinP: appConfig.OllamaMinP, }, } jsonPayload, err = json.Marshal(ollamaRequest) if err != nil { - return "", err } @@ -683,7 +733,6 @@ func DoOllamaRequest( request, err := http.NewRequest(http.MethodPost, appConfig.Endpoint, bytes.NewBuffer(jsonPayload)) if err != nil { - return "", err } @@ -715,16 +764,14 @@ func DoOllamaRequest( }, } } - response, err := httpClient.Do(request) + response, err := httpClient.Do(request) if err != nil { return "", err } defer response.Body.Close() - log.Println("response body:", response.Body) - var ollamaChatResponse OllamaChatMessagesResponse err = json.NewDecoder(response.Body).Decode(&ollamaChatResponse) @@ -732,6 +779,8 @@ func DoOllamaRequest( return "", err } + log.Println("ollama chat response: ", ollamaChatResponse) + return ollamaChatResponse.Messages.Content, nil } @@ -740,9 +789,9 @@ func OllamaRequestProcessor( client *girc.Client, event girc.Event, ollamaMemory *[]MemoryElement, - prompt string, + prompt, systemPrompt string, ) string { - response, err := DoOllamaRequest(appConfig, ollamaMemory, prompt) + response, err := DoOllamaRequest(appConfig, ollamaMemory, prompt, systemPrompt) if err != nil { client.Cmd.ReplyTo(event, "error: "+err.Error()) @@ -807,7 +856,7 @@ func OllamaHandler( return } - result := OllamaRequestProcessor(appConfig, client, event, ollamaMemory, prompt) + result := OllamaRequestProcessor(appConfig, client, event, ollamaMemory, prompt, appConfig.SystemPrompt) if result != "" { SendToIRC(client, event, result, appConfig.ChromaFormatter) } @@ -822,6 +871,7 @@ func (t *ProxyRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) if err != nil { return nil, err } + transport.Proxy = http.ProxyURL(proxyURL) } @@ -841,7 +891,7 @@ func (t *ProxyRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) func DoGeminiRequest( appConfig *TomlConfig, geminiMemory *[]*genai.Content, - prompt string, + prompt, systemPrompt string, ) (string, error) { httpProxyClient := &http.Client{Transport: &ProxyRoundTripper{ APIKey: appConfig.Apikey, @@ -853,15 +903,37 @@ func DoGeminiRequest( clientGemini, err := genai.NewClient(ctx, option.WithHTTPClient(httpProxyClient)) if err != nil { - - return "", err + return "", fmt.Errorf("Could not create a genai client.", err) } defer clientGemini.Close() model := clientGemini.GenerativeModel(appConfig.Model) - model.SetTemperature(float32(appConfig.Temp)) + model.SetTemperature(float32(appConfig.Temperature)) model.SetTopK(appConfig.TopK) model.SetTopP(appConfig.TopP) + model.SystemInstruction = &genai.Content{ + Parts: []genai.Part{ + genai.Text(systemPrompt), + }, + } + model.SafetySettings = []*genai.SafetySetting{ + { + Category: genai.HarmCategoryDangerousContent, + Threshold: genai.HarmBlockNone, + }, + { + Category: genai.HarmCategoryHarassment, + Threshold: genai.HarmBlockNone, + }, + { + Category: genai.HarmCategoryHateSpeech, + Threshold: genai.HarmBlockNone, + }, + { + Category: genai.HarmCategorySexuallyExplicit, + Threshold: genai.HarmBlockNone, + }, + } cs := model.StartChat() @@ -869,8 +941,7 @@ func DoGeminiRequest( resp, err := cs.SendMessage(ctx, genai.Text(prompt)) if err != nil { - - return "", err + return "", fmt.Errorf("Gemini: Could not send message", err) } return returnGeminiResponse(resp), nil @@ -881,9 +952,9 @@ func GeminiRequestProcessor( client *girc.Client, event girc.Event, geminiMemory *[]*genai.Content, - prompt string, + prompt, systemPrompt string, ) string { - geminiResponse, err := DoGeminiRequest(appConfig, geminiMemory, prompt) + geminiResponse, err := DoGeminiRequest(appConfig, geminiMemory, prompt, systemPrompt) if err != nil { client.Cmd.ReplyTo(event, "error: "+err.Error()) @@ -969,7 +1040,7 @@ func GeminiHandler( return } - result := GeminiRequestProcessor(appConfig, client, event, geminiMemory, prompt) + result := GeminiRequestProcessor(appConfig, client, event, geminiMemory, prompt, appConfig.SystemPrompt) if result != "" { SendToIRC(client, event, result, appConfig.ChromaFormatter) @@ -980,7 +1051,7 @@ func GeminiHandler( func DoChatGPTRequest( appConfig *TomlConfig, gptMemory *[]openai.ChatCompletionMessage, - prompt string, + prompt, systemPrompt string, ) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.RequestTimeout)*time.Second) defer cancel() @@ -1011,6 +1082,7 @@ func DoChatGPTRequest( config := openai.DefaultConfig(appConfig.Apikey) config.HTTPClient = &httpClient + if appConfig.Endpoint != "" { config.BaseURL = appConfig.Endpoint log.Print(config.BaseURL) @@ -1019,6 +1091,11 @@ func DoChatGPTRequest( gptClient := openai.NewClientWithConfig(config) *gptMemory = append(*gptMemory, openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleSystem, + Content: systemPrompt, + }) + + *gptMemory = append(*gptMemory, openai.ChatCompletionMessage{ Role: openai.ChatMessageRoleUser, Content: prompt, }) @@ -1028,7 +1105,6 @@ func DoChatGPTRequest( Messages: *gptMemory, }) if err != nil { - return "", err } @@ -1040,9 +1116,9 @@ func ChatGPTRequestProcessor( client *girc.Client, event girc.Event, gptMemory *[]openai.ChatCompletionMessage, - prompt string, + prompt, systemPrompt string, ) string { - resp, err := DoChatGPTRequest(appConfig, gptMemory, prompt) + resp, err := DoChatGPTRequest(appConfig, gptMemory, prompt, systemPrompt) if err != nil { client.Cmd.ReplyTo(event, "error: "+err.Error()) @@ -1115,14 +1191,16 @@ func ChatGPTHandler( return } - result := ChatGPTRequestProcessor(appConfig, client, event, gptMemory, prompt) + result := ChatGPTRequestProcessor(appConfig, client, event, gptMemory, prompt, appConfig.SystemPrompt) if result != "" { SendToIRC(client, event, result, appConfig.ChromaFormatter) } }) } -func connectToDB(appConfig *TomlConfig, ctx *context.Context, poolChan chan *pgxpool.Pool) { +func connectToDB(appConfig *TomlConfig, ctx *context.Context, irc *girc.Client) { + var pool *pgxpool.Pool + dbURL := fmt.Sprintf( "postgres://%s:%s@%s/%s", appConfig.DatabaseUser, @@ -1134,40 +1212,54 @@ func connectToDB(appConfig *TomlConfig, ctx *context.Context, poolChan chan *pgx poolConfig, err := pgxpool.ParseConfig(dbURL) if err != nil { - LogErrorFatal(err) + LogError(err) + + return } - pool, err := pgxpool.NewWithConfig(*ctx, poolConfig) + dbConnect := func() (*pgxpool.Pool, error) { + return pgxpool.NewWithConfig(*ctx, poolConfig) + } + + pool, err = backoff.Retry(*ctx, dbConnect, backoff.WithBackOff(backoff.NewExponentialBackOff())) if err != nil { - LogErrorFatal(err) - } else { - log.Printf("%s connected to database", appConfig.IRCDName) - - for _, channel := range appConfig.ScrapeChannels { - tableName := getTableFromChanName(channel[0], appConfig.IRCDName) - query := fmt.Sprintf( - `create table if not exists %s ( - id serial primary key, - channel text not null, - log text not null, - nick text not null, - dateadded timestamp default current_timestamp - )`, tableName) - - _, err = pool.Exec(*ctx, query) - if err != nil { - LogErrorFatal(err) - } - } + LogError(err) + } + + log.Printf("%s connected to database", appConfig.IRCDName) - appConfig.pool = pool - poolChan <- pool + for _, channel := range appConfig.ScrapeChannels { + tableName := getTableFromChanName(channel[0], appConfig.IRCDName) + query := fmt.Sprintf( + `create table if not exists %s ( + id serial primary key, + channel text not null, + log text not null, + nick text not null, + dateadded timestamp default current_timestamp + )`, tableName) + + _, err := pool.Exec(*ctx, query) + if err != nil { + LogError(err) + + continue + } } + + appConfig.pool = pool } -func scrapeChannel(irc *girc.Client, poolChan chan *pgxpool.Pool, appConfig TomlConfig) { +func scrapeChannel(irc *girc.Client, appConfig *TomlConfig) { + log.Print("spawning scraper") + irc.Handlers.AddBg(girc.PRIVMSG, func(_ *girc.Client, event girc.Event) { - pool := <-poolChan + if appConfig.pool == nil { + log.Println("no db connection. cant write scrapes to db.") + + return + } + tableName := getTableFromChanName(event.Params[0], appConfig.IRCDName) query := fmt.Sprintf( "insert into %s (channel,log,nick) values ('%s','%s','%s')", @@ -1177,7 +1269,12 @@ func scrapeChannel(irc *girc.Client, poolChan chan *pgxpool.Pool, appConfig Toml event.Source.Name, ) - _, err := pool.Exec(context.Background(), query) + log.Println(query) + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.RequestTimeout)*time.Second) + defer cancel() + + _, err := appConfig.pool.Exec(ctx, query) if err != nil { LogError(err) } @@ -1202,7 +1299,6 @@ func populateWatchListWords(appConfig *TomlConfig) { appConfig.WatchLists[watchlistName] = watchlist } } - } func WatchListHandler(irc *girc.Client, appConfig TomlConfig) { @@ -1269,8 +1365,6 @@ func runIRC(appConfig TomlConfig) { var ORMemory []MemoryElement - poolChan := make(chan *pgxpool.Pool, 1) - irc := girc.New(girc.Config{ Server: appConfig.IrcServer, Port: appConfig.IrcPort, @@ -1332,7 +1426,7 @@ func runIRC(appConfig TomlConfig) { irc.Config.TLSConfig.Certificates = []tls.Certificate{cert} } - irc.Handlers.AddBg(girc.CONNECTED, func(c *girc.Client, _ girc.Event) { + irc.Handlers.AddBg(girc.CONNECTED, func(_ *girc.Client, _ girc.Event) { for _, channel := range appConfig.IrcChannels { IrcJoin(irc, channel) } @@ -1385,21 +1479,21 @@ func runIRC(appConfig TomlConfig) { context, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.RequestTimeout)*time.Second) defer cancel() - go connectToDB(&appConfig, &context, poolChan) + go connectToDB(&appConfig, &context, irc) } if len(appConfig.ScrapeChannels) > 0 { - irc.Handlers.AddBg(girc.CONNECTED, func(c *girc.Client, _ girc.Event) { + irc.Handlers.AddBg(girc.CONNECTED, func(_ *girc.Client, _ girc.Event) { for _, channel := range appConfig.ScrapeChannels { IrcJoin(irc, channel) } }) - go scrapeChannel(irc, poolChan, appConfig) + go scrapeChannel(irc, &appConfig) } if len(appConfig.WatchLists) > 0 { - irc.Handlers.AddBg(girc.CONNECTED, func(client *girc.Client, _ girc.Event) { + irc.Handlers.AddBg(girc.CONNECTED, func(_ *girc.Client, _ girc.Event) { for _, watchlist := range appConfig.WatchLists { log.Print("joining ", watchlist.AlertChannel) IrcJoin(irc, watchlist.AlertChannel) @@ -1417,36 +1511,51 @@ func runIRC(appConfig TomlConfig) { if len(appConfig.Rss) > 0 { irc.Handlers.AddBg(girc.CONNECTED, func(client *girc.Client, _ girc.Event) { - go runRSS(&appConfig, irc) + for _, rss := range appConfig.Rss { + log.Print("RSS: joining ", rss.Channel) + IrcJoin(irc, rss.Channel) + } }) - } - for { - var dialer proxy.Dialer + go runRSS(&appConfig, irc) + } - if appConfig.IRCProxy != "" { - proxyURL, err := url.Parse(appConfig.IRCProxy) - if err != nil { - LogErrorFatal(err) - } + var dialer proxy.Dialer - dialer, err = proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(appConfig.RequestTimeout) * time.Second}) - if err != nil { - LogErrorFatal(err) - } + if appConfig.IRCProxy != "" { + proxyURL, err := url.Parse(appConfig.IRCProxy) + if err != nil { + LogErrorFatal(err) } - if err := irc.DialerConnect(dialer); err != nil { - LogError(err) - log.Println("reconnecting in " + strconv.Itoa(appConfig.MillaReconnectDelay)) - time.Sleep(time.Duration(appConfig.MillaReconnectDelay) * time.Second) - } else { - return + dialer, err = proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(appConfig.RequestTimeout) * time.Second}) + if err != nil { + LogErrorFatal(err) } } + + connectToIRC := func() (string, error) { + return "", irc.DialerConnect(dialer) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.MillaReconnectDelay)*time.Second) + defer cancel() + + _, err := backoff.Retry(ctx, connectToIRC, backoff.WithBackOff(backoff.NewExponentialBackOff())) + if err != nil { + LogError(err) + } else { + return + } +} + +func goroutines() interface{} { + return runtime.NumGoroutine() } func main() { + expvar.Publish("Goroutines", expvar.Func(goroutines)) + quitChannel := make(chan os.Signal, 1) signal.Notify(quitChannel, syscall.SIGINT, syscall.SIGTERM) @@ -1480,5 +1589,10 @@ func main() { go runIRC(v) } + go func() { + err := http.ListenAndServe(":6060", nil) + log.Println(err) + }() + <-quitChannel } diff --git a/openrouter.go b/openrouter.go index 09e78b5..7ae5d62 100644 --- a/openrouter.go +++ b/openrouter.go @@ -45,6 +45,7 @@ func DoORRequest( ollamaRequest := OllamaChatRequest{ Model: appConfig.Model, + System: appConfig.SystemPrompt, Messages: *memory, } @@ -218,10 +218,39 @@ func registerLuaCommand(luaState *lua.LState, appConfig *TomlConfig) func(*lua.L } } +func registerTriggeredScript(luaState *lua.LState, appConfig *TomlConfig) func(*lua.LState) int { + return func(luaState *lua.LState) int { + path := luaState.CheckString(1) + funcName := luaState.CheckString(3) //nolint: mnd,gomnd + eventTypesTable := luaState.CheckTable(2) //nolint: mnd,gomnd + + _, ok := appConfig.TriggeredScripts[path] + if ok { + log.Print("triggered script already registered: ", path) + + return 0 + } + + var eventTypes []string + + eventTypesTable.ForEach(func(_, value lua.LValue) { + if value.Type() != lua.LTString { + log.Print("event type for TriggeredScripts must be a string") + } else { + eventTypes = append(eventTypes, value.String()) + } + }) + + appConfig.insertTriggeredScript(path, funcName, eventTypes) + + return 0 + } +} + func ircJoinChannelClosure(luaState *lua.LState, client *girc.Client) func(*lua.LState) int { return func(luaState *lua.LState) int { channel := luaState.CheckString(1) - password := luaState.CheckString(2) + password := luaState.CheckString(2) //nolint: mnd,gomnd if password != "" { client.Cmd.JoinKey(channel, password) @@ -261,8 +290,9 @@ func orRequestClosure(luaState *lua.LState, appConfig *TomlConfig) func(*lua.LSt func ollamaRequestClosure(luaState *lua.LState, appConfig *TomlConfig) func(*lua.LState) int { return func(luaState *lua.LState) int { prompt := luaState.CheckString(1) + systemPrompt := luaState.CheckString(2) //nolint: mnd,gomnd - result, err := DoOllamaRequest(appConfig, &[]MemoryElement{}, prompt) + result, err := DoOllamaRequest(appConfig, &[]MemoryElement{}, prompt, systemPrompt) if err != nil { LogError(err) } @@ -276,8 +306,9 @@ func ollamaRequestClosure(luaState *lua.LState, appConfig *TomlConfig) func(*lua func geminiRequestClosure(luaState *lua.LState, appConfig *TomlConfig) func(*lua.LState) int { return func(luaState *lua.LState) int { prompt := luaState.CheckString(1) + systemPrompt := luaState.CheckString(2) //nolint: mnd,gomnd - result, err := DoGeminiRequest(appConfig, &[]*genai.Content{}, prompt) + result, err := DoGeminiRequest(appConfig, &[]*genai.Content{}, prompt, systemPrompt) if err != nil { LogError(err) } @@ -291,8 +322,9 @@ func geminiRequestClosure(luaState *lua.LState, appConfig *TomlConfig) func(*lua func chatGPTRequestClosure(luaState *lua.LState, appConfig *TomlConfig) func(*lua.LState) int { return func(luaState *lua.LState) int { prompt := luaState.CheckString(1) + systemPrompt := luaState.CheckString(2) //nolint: mnd,gomnd - result, err := DoChatGPTRequest(appConfig, &[]openai.ChatCompletionMessage{}, prompt) + result, err := DoChatGPTRequest(appConfig, &[]openai.ChatCompletionMessage{}, prompt, systemPrompt) if err != nil { LogError(err) } @@ -341,6 +373,7 @@ func urlEncode(luaState *lua.LState) func(*lua.LState) int { URL := luaState.CheckString(1) escapedURL := url.QueryEscape(URL) luaState.Push(lua.LString(escapedURL)) + return 1 } } @@ -364,6 +397,9 @@ func millaModuleLoaderClosure(luaState *lua.LState, client *girc.Client, appConf registerStructAsLuaMetaTable[TomlConfig](luaState, millaModule, checkStruct, TomlConfig{}, "toml_config") registerStructAsLuaMetaTable[CustomCommand](luaState, millaModule, checkStruct, CustomCommand{}, "custom_command") registerStructAsLuaMetaTable[LogModel](luaState, millaModule, checkStruct, LogModel{}, "log_model") + registerStructAsLuaMetaTable[girc.Source](luaState, millaModule, checkStruct, girc.Source{}, "girc_source") + // FIXME: throws error + // registerStructAsLuaMetaTable[girc.Tags](luaState, millaModule, checkStruct, girc.Tags{}, "girc_tags") registerStructAsLuaMetaTable[girc.Event](luaState, millaModule, checkStruct, girc.Event{}, "girc_event") luaState.SetGlobal("milla", millaModule) @@ -411,6 +447,7 @@ func RunScript(scriptPath string, client *girc.Client, appConfig *TomlConfig) { if err != nil { LogError(err) } + proxyTransport.Proxy = http.ProxyURL(proxyURL) } @@ -477,6 +514,7 @@ func RunLuaFunc( if err != nil { LogError(err) } + proxyTransport.Proxy = http.ProxyURL(proxyURL) } @@ -498,6 +536,7 @@ func RunLuaFunc( log.Print(cmd) log.Print(args) + if err := luaState.CallByParam(funcLValue, lua.LString(args)); err != nil { log.Print("failed running lua command ...") LogError(err) @@ -510,3 +549,94 @@ func RunLuaFunc( return result.String() } + +func RunTriggeredLuaFunc( + funcName, scriptPath string, + client *girc.Client, + event girc.Event, + appConfig *TomlConfig, +) string { + luaState := lua.NewState() + defer luaState.Close() + + ctx, cancel := context.WithCancel(context.Background()) + + luaState.SetContext(ctx) + + appConfig.insertLState(scriptPath, luaState, cancel) + + luaState.PreloadModule("milla", millaModuleLoaderClosure(luaState, client, appConfig)) + gluasocket.Preload(luaState) + gluaxmlpath.Preload(luaState) + luaState.PreloadModule("yaml", gluayaml.Loader) + luaState.PreloadModule("re", gluare.Loader) + luaState.PreloadModule("json", gopherjson.Loader) + + var proxyString string + if os.Getenv("ALL_PROXY") != "" { + proxyString = os.Getenv("ALL_PROXY") + } else if os.Getenv("HTTPS_PROXY") != "" { + proxyString = os.Getenv("HTTPS_PROXY") + } else if os.Getenv("HTTP_PROXY") != "" { + proxyString = os.Getenv("HTTP_PROXY") + } else if os.Getenv("https_proxy") != "" { + proxyString = os.Getenv("https_proxy") + } else if os.Getenv("http_proxy") != "" { + proxyString = os.Getenv("http_proxy") + } + + log.Print("set proxy env to:", proxyString) + + proxyTransport := &http.Transport{} + + if proxyString != "" { + proxyURL, err := url.Parse(proxyString) + if err != nil { + LogError(err) + } + + proxyTransport.Proxy = http.ProxyURL(proxyURL) + } + + luaState.PreloadModule("http", gluahttp.NewHttpModule(&http.Client{Transport: proxyTransport}).Loader) + + log.Print("Running lua command script: ", scriptPath) + + if err := luaState.DoFile(scriptPath); err != nil { + LogError(err) + + return "" + } + + funcLValue := lua.P{ + Fn: luaState.GetGlobal(funcName), + NRet: 1, + Protect: true, + } + + if err := luaState.CallByParam(funcLValue); err != nil { + log.Print("failed running lua command ...") + LogError(err) + + return "" + } + + result := luaState.Get(-1) + luaState.Pop(1) + + return result.String() +} + +func registerTriggeredScripts(irc *girc.Client, appConfig TomlConfig) { + for _, triggeredScript := range appConfig.TriggeredScripts { + for _, triggerType := range triggeredScript.TriggerType { + switch triggerType { + case girc.PRIVMSG: + irc.Handlers.AddBg(girc.PRIVMSG, func(_ *girc.Client, event girc.Event) { + RunTriggeredLuaFunc(triggeredScript.FuncName, triggeredScript.Path, irc, event, &appConfig) + }) + default: + } + } + } +} diff --git a/plugins/urban.lua b/plugins/urban.lua index 7b7e71c..aaab985 100644 --- a/plugins/urban.lua +++ b/plugins/urban.lua @@ -65,7 +65,8 @@ function milla_urban(cli_args) for _, v in ipairs(json_response["list"]) do for kk, vv in pairs(v) do print(kk, vv) end if count > 0 then - result = result .. tostring(count) .. v["definition"] .. "----" + result = result .. tostring(count) .. " - " .. v["definition"] .. + " ---- " end count = count - 1 end @@ -26,12 +26,16 @@ func GetFeed(feed FeedConfig, ) { rowName := groupName + "__" + feed.Name + "__" - parsedFeed, err := feed.FeedParser.ParseURL(feed.URL) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(feed.Timeout)*time.Second) + defer cancel() + + parsedFeed, err := feed.FeedParser.ParseURLWithContext(feed.URL, ctx) if err != nil { LogError(err) } else { if len(parsedFeed.Items) > 0 { query := fmt.Sprintf("select newest_unix_time from rss where name = '%s'", rowName) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) defer cancel() @@ -39,7 +43,12 @@ func GetFeed(feed FeedConfig, err := pool.QueryRow(ctx, query).Scan(&newestFromDB) if err != nil { - pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName)) + _, err = pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName)) + if err != nil { + LogError(err) + + return + } } sortFunc := func(a, b *gofeed.Item) int { @@ -56,7 +65,7 @@ func GetFeed(feed FeedConfig, for _, item := range parsedFeed.Items { if item.PublishedParsed.Unix() > newestFromDB { - client.Cmd.Message(channel[0], parsedFeed.Title+": "+item.Title+">>>"+item.Link) + client.Cmd.Message(channel[0], parsedFeed.Title+": "+item.Title+" >>> "+item.Link) } } @@ -69,7 +78,6 @@ func GetFeed(feed FeedConfig, if err != nil { LogError(err) } - } } } @@ -83,41 +91,45 @@ func feedDispatcher( period int, ) { for { - for i := range len(config.Feeds) { - config.Feeds[i].FeedParser = gofeed.NewParser() + if client.IsConnected() { + for i := range len(config.Feeds) { + config.Feeds[i].FeedParser = gofeed.NewParser() - config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent + config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent - if config.Feeds[i].Proxy != "" { - proxyURL, err := url.Parse(config.Feeds[i].Proxy) - if err != nil { - LogError(err) + if config.Feeds[i].Proxy != "" { + proxyURL, err := url.Parse(config.Feeds[i].Proxy) + if err != nil { + LogError(err) - continue - } + continue + } - dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second}) - if err != nil { - LogError(err) + dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second}) + if err != nil { + LogError(err) - continue - } + continue + } - httpClient := http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - }, + httpClient := http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + }, + } + + config.Feeds[i].FeedParser.Client = &httpClient } + } - config.Feeds[i].FeedParser.Client = &httpClient + for _, feed := range config.Feeds { + go GetFeed(feed, client, pool, channel, groupName) } - } - for _, feed := range config.Feeds { - go GetFeed(feed, client, pool, channel, groupName) + time.Sleep(time.Duration(period) * time.Second) + } else { + time.Sleep(time.Duration(10) * time.Second) } - - time.Sleep(time.Duration(period) * time.Second) } } @@ -152,21 +164,24 @@ func runRSS(appConfig *TomlConfig, client *girc.Client) { )`) for { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) - defer cancel() + if appConfig.pool != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancel() - _, err := appConfig.pool.Exec(ctx, query) - if err != nil { - LogError(err) - time.Sleep(time.Duration(60) * time.Second) - } else { - break + _, err := appConfig.pool.Exec(ctx, query) + if err != nil { + LogError(err) + } else { + break + } } + + time.Sleep(time.Duration(10) * time.Second) } + log.Print("spawning the RSS feed dispatcher") + for groupName, rss := range appConfig.Rss { - log.Print("RSS: joining ", rss.Channel) - IrcJoin(client, rss.Channel) rssConfig := ParseRSSConfig(rss.RssFile) if rssConfig == nil { log.Print("Could not parse RSS config file " + rss.RssFile + ". Exiting.") @@ -20,10 +20,11 @@ type LogModel struct { } type CustomCommand struct { - SQL string `toml:"sql"` - Limit int `toml:"limit"` - Context []string `toml:"context"` - Prompt string `toml:"prompt"` + SQL string `toml:"sql"` + Limit int `toml:"limit"` + Context []string `toml:"context"` + Prompt string `toml:"prompt"` + SystemPrompt string `toml:"systemPrompt"` } type LuaLstates struct { @@ -46,6 +47,13 @@ type LuaCommand struct { FuncName string } +type TriggeredScripts struct { + Path string + FuncName string + Channels [][]string + TriggerType []string +} + type RssFile struct { RssFile string `toml:"rssFile"` Channel []string `toml:"channel"` @@ -62,7 +70,6 @@ type TomlConfig struct { ChromaFormatter string `toml:"chromaFormatter"` Provider string `toml:"provider"` Apikey string `toml:"apikey"` - OllamaSystem string `toml:"ollamaSystem"` ClientCertPath string `toml:"clientCertPath"` ServerPass string `toml:"serverPass"` Bind string `toml:"bind"` @@ -83,12 +90,13 @@ type TomlConfig struct { AnthropicVersion string `toml:"anthropicVersion"` Plugins []string `toml:"plugins"` Context []string `toml:"context"` + SystemPrompt string `toml:"systemPrompt"` CustomCommands map[string]CustomCommand `toml:"customCommands"` WatchLists map[string]WatchList `toml:"watchList"` LuaStates map[string]LuaLstates LuaCommands map[string]LuaCommand + TriggeredScripts map[string]TriggeredScripts Rss map[string]RssFile `toml:"rss"` - Temp float64 `toml:"temp"` RequestTimeout int `toml:"requestTimeout"` MillaReconnectDelay int `toml:"millaReconnectDelay"` IrcPort int `toml:"ircPort"` @@ -96,6 +104,16 @@ type TomlConfig struct { MemoryLimit int `toml:"memoryLimit"` PingDelay int `toml:"pingDelay"` PingTimeout int `toml:"pingTimeout"` + OllamaMirostat int `json:"ollamaMirostat"` + OllamaMirostatEta float64 `json:"ollamaMirostatEta"` + OllamaMirostatTau float64 `json:"ollamaMirostatTau"` + OllamaNumCtx int `json:"ollamaNumCtx"` + OllamaRepeatLastN int `json:"ollamaRepeatLastN"` + OllamaRepeatPenalty float64 `json:"ollamaRepeatPenalty"` + Temperature float64 `json:"temperature"` + OllamaSeed int `json:"ollamaSeed"` + OllamaNumPredict int `json:"ollamaNumPredict"` + OllamaMinP float64 `json:"ollamaMinP"` TopP float32 `toml:"topP"` TopK int32 `toml:"topK"` EnableSasl bool `toml:"enableSasl"` @@ -153,12 +171,42 @@ func (config *TomlConfig) deleteLuaCommand(name string) { delete(config.LuaCommands, name) } +func (config *TomlConfig) insertTriggeredScript(path, cmd string, triggerType []string) { + if config.TriggeredScripts == nil { + config.TriggeredScripts = make(map[string]TriggeredScripts) + } + config.TriggeredScripts[path] = TriggeredScripts{ + Path: path, + FuncName: cmd, + TriggerType: triggerType, + } +} + +func (config *TomlConfig) deleteTriggeredScript(name string) { + if config.TriggeredScripts == nil { + return + } + + delete(config.TriggeredScripts, name) +} + type AppConfig struct { Ircd map[string]TomlConfig `toml:"ircd"` } type OllamaRequestOptions struct { - Temperature float64 `json:"temperature"` + Mirostat int `json:"mirostat"` + MirostatEta float64 `json:"mirostat_eta"` + MirostatTau float64 `json:"mirostat_tau"` + NumCtx int `json:"num_ctx"` + RepeatLastN int `json:"repeat_last_n"` + RepeatPenalty float64 `json:"repeat_penalty"` + Temperature float64 `json:"temperature"` + Seed int `json:"seed"` + NumPredict int `json:"num_predict"` + TopK int32 `json:"top_k"` + TopP float32 `json:"top_p"` + MinP float64 `json:"min_p"` } type OllamaChatResponse struct { @@ -175,6 +223,7 @@ type OllamaChatRequest struct { Stream bool `json:"stream"` KeepAlive time.Duration `json:"keep_alive"` Options OllamaRequestOptions `json:"options"` + System string `json:"system"` Messages []MemoryElement `json:"messages"` } |