services/feeds/handler/crawl.go

package handler

import (
	"context"
	"crypto/md5"
	"fmt"

	"net/url"

	"github.com/SlyMarbo/rss"
	log "github.com/micro/micro/v3/service/logger"
	"github.com/micro/services/feeds/parser"
	feeds "github.com/micro/services/feeds/proto"
	posts "github.com/micro/services/posts/proto"
)

func (e *Feeds) fetchAll() {
	fs := []*feeds.Feed{}
	err := e.feeds.Read(e.feedsNameIndex.ToQuery(nil), &fs)
	if err != nil {
		log.Errorf("Error listing feeds: %v", err)
		return
	}
	if len(fs) == 0 {
		log.Infof("No feeds to fetch")
		return
	}
	for _, feed := range fs {
		err = e.fetch(feed)
		if err != nil {
			log.Errorf("Error saving post: %v", err)
		}
	}
}

func (e *Feeds) fetch(f *feeds.Feed) error {
	url := f.Url

	log.Infof("Fetching address %v", url)
	fd, err := rss.Fetch(url)
	if err != nil {
		return fmt.Errorf("Error fetching address %v: %v", url, err)
	}
	domain := getDomain(url)

	for _, item := range fd.Items {
		id := fmt.Sprintf("%x", md5.Sum([]byte(item.ID)))

		err = e.entries.Create(feeds.Entry{
			Id:       id,
			Url:      item.Link,
			Title:    item.Title,
			Domain:   domain,
			Content:  item.Summary,
			Date:     item.Date.Unix(),
			Category: f.Category,
		})
		if err != nil {
			return fmt.Errorf("Error saving item: %v", err)
		}

		var tags []string
		if len(f.Category) > 0 {
			tags = append(tags, f.Category)
		}

		// check if content exists
		content := item.Content
		if len(content) == 0 && len(item.Summary) > 0 {
			content = item.Summary
		}

		// if we have a parser which returns content use it
		// e.g cnbc
		c, err := parser.Parse(item.Link)
		if err == nil && len(c) > 0 {
			content = c
		}

		// @todo make this optional
		_, err = e.postsService.Save(context.TODO(), &posts.SaveRequest{
			Id:        id,
			Title:     item.Title,
			Content:   content,
			Timestamp: item.Date.Unix(),
			Metadata: map[string]string{
				"domain": domain,
				"link":   item.Link,
			},
			Tags: tags,
		})
		if err != nil {
			return err
		}
	}

	return nil
}

func getDomain(address string) string {
	uri, _ := url.Parse(address)
	return uri.Host
}