diff options
Diffstat (limited to '')
| -rw-r--r-- | rss.go | 158 |
1 files changed, 78 insertions, 80 deletions
@@ -19,7 +19,6 @@ import ( ) func GetFeed(feed FeedConfig, - feedChanel chan<- *gofeed.Feed, client *girc.Client, pool *pgxpool.Pool, channel, groupName string, @@ -28,54 +27,52 @@ func GetFeed(feed FeedConfig, parsedFeed, err := feed.FeedParser.ParseURL(feed.URL) if err != nil { - log.Print(err) + LogError(err) } else { - query := fmt.Sprintf("select newest_unix_time from rss where name = '%s'", rowName) - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) - defer cancel() + if len(parsedFeed.Items) > 0 { + query := fmt.Sprintf("select newest_unix_time from rss where name = '%s'", rowName) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancel() - newestFromDB := int64(0) + newestFromDB := int64(0) - err := pool.QueryRow(ctx, query).Scan(&newestFromDB) - if err != nil { - pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName)) - } + err := pool.QueryRow(ctx, query).Scan(&newestFromDB) + if err != nil { + pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName)) + } - log.Print("Newset from DB: ", newestFromDB) + log.Print("Newset from DB: ", newestFromDB) - sortFunc := func(a, b *gofeed.Item) int { - if a.PublishedParsed.Before(*b.PublishedParsed) { - return -1 - } else if a.PublishedParsed.After(*b.PublishedParsed) { - return 1 - } + sortFunc := func(a, b *gofeed.Item) int { + if a.PublishedParsed.Before(*b.PublishedParsed) { + return -1 + } else if a.PublishedParsed.After(*b.PublishedParsed) { + return 1 + } - return 0 - } + return 0 + } - slices.SortFunc(parsedFeed.Items, sortFunc) + slices.SortFunc(parsedFeed.Items, sortFunc) - for _, item := range parsedFeed.Items { - if item.PublishedParsed.Unix() > newestFromDB { - client.Cmd.Message(channel, parsedFeed.Title+": "+item.Title) + for _, item := range parsedFeed.Items { + if item.PublishedParsed.Unix() > newestFromDB { + client.Cmd.Message(channel, parsedFeed.Title+": "+item.Title+">>>"+item.Link) + } } - } - log.Print(parsedFeed.Items[0].PublishedParsed.Unix()) - log.Print(parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix()) + query = fmt.Sprintf("update rss set newest_unix_time = %d where name = '%s'", parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix(), rowName) - query = fmt.Sprintf("update rss set newest_unix_time = %d where name = '%s'", parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix(), rowName) + ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancel() - ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) - defer cancel() + _, err = pool.Exec(ctx2, query) + if err != nil { + LogError(err) + } - _, err = pool.Exec(ctx2, query) - if err != nil { - log.Print(err) } } - - feedChanel <- parsedFeed } func feedDispatcher( @@ -83,49 +80,51 @@ func feedDispatcher( client *girc.Client, pool *pgxpool.Pool, channel, groupName string, + period int, ) { - feedChanel := make(chan *gofeed.Feed) + for { + for i := range len(config.Feeds) { + config.Feeds[i].FeedParser = gofeed.NewParser() - for i := range len(config.Feeds) { - config.Feeds[i].FeedParser = gofeed.NewParser() + config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent - config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent + if config.Feeds[i].Proxy != "" { + proxyURL, err := url.Parse(config.Feeds[i].Proxy) + if err != nil { + LogError(err) - if config.Feeds[i].Proxy != "" { - proxyURL, err := url.Parse(config.Feeds[i].Proxy) - if err != nil { - log.Print(err) - continue - } + continue + } - dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second}) - if err != nil { - log.Print(err) + dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second}) + if err != nil { + LogError(err) - continue - } + continue + } - httpClient := http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - }, + httpClient := http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + }, + } + + config.Feeds[i].FeedParser.Client = &httpClient } + } - config.Feeds[i].FeedParser.Client = &httpClient + for _, feed := range config.Feeds { + go GetFeed(feed, client, pool, channel, groupName) } - } - for _, feed := range config.Feeds { - go GetFeed(feed, feedChanel, client, pool, channel, groupName) + time.Sleep(time.Duration(period) * time.Second) } - - // <-feedChanel } func ParseRSSConfig(rssConfFilePath string) *RSSConfig { file, err := os.Open(rssConfFilePath) if err != nil { - log.Print(err) + LogError(err) return nil } @@ -136,7 +135,7 @@ func ParseRSSConfig(rssConfFilePath string) *RSSConfig { err = decoder.Decode(&config) if err != nil { - log.Print(err) + LogError(err) return nil } @@ -145,35 +144,34 @@ func ParseRSSConfig(rssConfFilePath string) *RSSConfig { } func runRSS(appConfig *TomlConfig, client *girc.Client) { - for { - query := fmt.Sprintf( - `create table if not exists rss ( + query := fmt.Sprintf( + `create table if not exists rss ( id serial primary key, name text not null unique, newest_unix_time bigint not null )`) - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.RequestTimeout)*time.Second) + for { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) defer cancel() _, err := appConfig.pool.Exec(ctx, query) if err != nil { - log.Print(err) - time.Sleep(time.Duration(appConfig.MillaReconnectDelay) * time.Second) + LogError(err) + time.Sleep(time.Duration(60) * time.Second) } else { - for groupName, rss := range appConfig.Rss { - log.Print("RSS: joining ", rss.Channel) - client.Cmd.Join(rss.Channel) - rssConfig := ParseRSSConfig(rss.RssFile) - if rssConfig == nil { - log.Print("Could not parse RSS config file " + rss.RssFile + ". Exiting.") - } else { - for { - feedDispatcher(*rssConfig, client, appConfig.pool, rss.Channel, groupName) - time.Sleep(time.Duration(rssConfig.Period) * time.Second) - } - } - } + break + } + } + + for groupName, rss := range appConfig.Rss { + log.Print("RSS: joining ", rss.Channel) + client.Cmd.Join(rss.Channel) + rssConfig := ParseRSSConfig(rss.RssFile) + if rssConfig == nil { + log.Print("Could not parse RSS config file " + rss.RssFile + ". Exiting.") + } else { + go feedDispatcher(*rssConfig, client, appConfig.pool, rss.Channel, groupName, rssConfig.Period) } } } |