Autogenerate services.m3o.com (#37)

* Autogenerate services.m3o.com * Openapi for all * Gen * Fix * Whaat * Fix dep * Fix * Hmm * Install make * Debug * Debug 1 * Location -> locations * Fix * Intall protoc gen micro * F * F * F * Push * Rename secret * Fix npm install * Fix script * Fix v2 * Ignore errors * Ignore v2 * F * F * F * Docs index * Add hugo theme * Hugo tania fixes * Change gen * Change gen 2 * Install hugo * Change gen * Gen fix * Change hugo install * Change hugo install * CNAME * Change articles wording * Tiny fix * Fix gen * Redoc it all * Fix gen * Fixing up protos * Fix proto * Fix gen * Fix * Trigger build * Fix copy * Openapi docs * Flatten * Changes * No date vol2 * Changes * Add make to chat * Fixes * Change * api spec * replace RSS * fix link * Dont continue on error * increase the width * use micro at master * change box colours * move some things * Pushing new readmes to see how they look like * Add skip file * Readmes * Nicer api link * Remove stutter * FIx mistake * set service font weight * Messages readme fix * add other font bold * Notes * Remove post from url * Revert "Remove post from url" This reverts commit 5fea2c23d0bafa910f5dc4d4cc63f71f578530e3. * move exampleSite to site * replace exampleSite with site * update readme * use filename for post * update index * Add source urls * set source as params * set source as params * Fix entries * Generator in go * Fixes to generator * F * Change doc gen * FIx cname * Fixing protos * Change to makefiles * Fix gen script Co-authored-by: Asim Aslam <asim@aslam.me>
2026-01-16 21:14:36 +00:00 · 2021-01-19 16:59:25 +00:00
parent ec81db0753
commit 40b71a9cf9
174 changed files with 10914 additions and 3157 deletions
--- a/feeds/handler/crawl.go
+++ b/feeds/handler/crawl.go
@@ -0,0 +1,77 @@
+package handler
+
+import (
+	"context"
+	"crypto/md5"
+	"fmt"
+
+	"net/url"
+
+	"github.com/SlyMarbo/rss"
+	log "github.com/micro/micro/v3/service/logger"
+	feeds "github.com/micro/services/feeds/proto"
+	posts "github.com/micro/services/posts/proto"
+)
+
+func (e *Feeds) fetchAll() {
+	fs := []*feeds.Feed{}
+	err := e.feeds.List(e.feedsNameIndex.ToQuery(nil), &fs)
+	if err != nil {
+		log.Errorf("Error listing feeds: %v", err)
+		return
+	}
+	if len(fs) == 0 {
+		log.Infof("No feeds to fetch")
+		return
+	}
+	for _, feed := range fs {
+		err = e.fetch(feed.Url)
+		if err != nil {
+			log.Errorf("Error saving post: %v", err)
+		}
+	}
+}
+
+func (e *Feeds) fetch(url string) error {
+	log.Infof("Fetching address %v", url)
+	fd, err := rss.Fetch(url)
+	if err != nil {
+		return fmt.Errorf("Error fetching address %v: %v", url, err)
+	}
+	domain := getDomain(url)
+
+	for _, item := range fd.Items {
+		id := fmt.Sprintf("%x", md5.Sum([]byte(item.ID)))
+		err = e.entries.Save(feeds.Entry{
+			Id:      id,
+			Url:     item.Link,
+			Title:   item.Title,
+			Domain:  domain,
+			Content: item.Summary,
+			Date:    item.Date.Unix(),
+		})
+		if err != nil {
+			return fmt.Errorf("Error saving item: %v", err)
+		}
+		// @todo make this optional
+		_, err := e.postsService.Save(context.TODO(), &posts.SaveRequest{
+			Id:        id,
+			Title:     item.Title,
+			Content:   item.Content,
+			Timestamp: item.Date.Unix(),
+			Metadata: map[string]string{
+				"domain": domain,
+				"link":   item.Link,
+			},
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getDomain(address string) string {
+	uri, _ := url.Parse(address)
+	return uri.Host
+}
--- a/feeds/handler/feeds.go
+++ b/feeds/handler/feeds.go
@@ -0,0 +1,93 @@
+package handler
+
+import (
+	"context"
+	"time"
+
+	"github.com/micro/dev/model"
+	log "github.com/micro/micro/v3/service/logger"
+	"github.com/micro/micro/v3/service/store"
+
+	feeds "github.com/micro/services/feeds/proto"
+	posts "github.com/micro/services/posts/proto"
+)
+
+type Feeds struct {
+	feeds            model.Model
+	entries          model.Model
+	postsService     posts.PostsService
+	feedsIdIndex     model.Index
+	feedsNameIndex   model.Index
+	entriesDateIndex model.Index
+	entriesURLIndex  model.Index
+}
+
+func NewFeeds(postsService posts.PostsService) *Feeds {
+	idIndex := model.ByEquality("url")
+	idIndex.Order.Type = model.OrderTypeUnordered
+
+	nameIndex := model.ByEquality("name")
+	nameIndex.Unique = true
+	nameIndex.Order.Type = model.OrderTypeUnordered
+
+	dateIndex := model.ByEquality("date")
+	dateIndex.Order.Type = model.OrderTypeDesc
+
+	entriesURLIndex := model.ByEquality("url")
+	entriesURLIndex.Order.Type = model.OrderTypeDesc
+	entriesURLIndex.Order.FieldName = "date"
+
+	f := &Feeds{
+		feeds: model.New(
+			store.DefaultStore,
+			"feeds",
+			model.Indexes(nameIndex),
+			&model.ModelOptions{
+				Debug:   false,
+				IdIndex: idIndex,
+			},
+		),
+		entries: model.New(
+			store.DefaultStore,
+			"entries",
+			model.Indexes(dateIndex, entriesURLIndex),
+			&model.ModelOptions{
+				Debug: false,
+			},
+		),
+		postsService:     postsService,
+		feedsIdIndex:     idIndex,
+		feedsNameIndex:   nameIndex,
+		entriesDateIndex: dateIndex,
+		entriesURLIndex:  entriesURLIndex,
+	}
+
+	go f.crawl()
+	return f
+}
+
+func (e *Feeds) crawl() {
+	e.fetchAll()
+	tick := time.NewTicker(1 * time.Minute)
+	for _ = range tick.C {
+		e.fetchAll()
+	}
+}
+
+func (e *Feeds) New(ctx context.Context, req *feeds.NewRequest, rsp *feeds.NewResponse) error {
+	log.Info("Received Feeds.New request")
+	e.feeds.Save(feeds.Feed{
+		Name: req.Name,
+		Url:  req.Url,
+	})
+	return nil
+}
+
+func (e *Feeds) Entries(ctx context.Context, req *feeds.EntriesRequest, rsp *feeds.EntriesResponse) error {
+	log.Info("Received Feeds.New request")
+	err := e.fetch(req.Url)
+	if err != nil {
+		return err
+	}
+	return e.entries.List(e.entriesURLIndex.ToQuery(req.Url), &rsp.Entries)
+}