automod: fix automod account age (#713)

these are cleanups related to an earlier bug when rolling out account
creation timestamp handling in automod

automod/engine/account_meta.go

···
 type AccountPrivate struct {
 	Email          string
 	EmailConfirmed bool
-	IndexedAt      time.Time
+	IndexedAt      *time.Time
 	AccountTags    []string
 }

automod/engine/engine_ozone.go

···
 		return nil, err
 	}
 	if creatorIdent == nil {
-		return nil, fmt.Errorf("identity not found for DID: %s", evt.CreatedBy)
+		return nil, fmt.Errorf("identity not found for creator DID: %s", evt.CreatedBy)
 	}
 	creatorMeta, err := eng.GetAccountMeta(ctx, creatorIdent)
 	if err != nil {
···
 		return nil, err
 	}
 	if subjectIdent == nil {
-		return nil, fmt.Errorf("identity not found for DID: %s", evt.SubjectDID)
+		return nil, fmt.Errorf("identity not found for subject DID: %s", evt.SubjectDID)
 	}
 	accountMeta, err := eng.GetAccountMeta(ctx, subjectIdent)
 	if err != nil {

automod/engine/fetch_account_meta.go

···
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"time"
 
 	comatproto "github.com/bluesky-social/indigo/api/atproto"
 	appbsky "github.com/bluesky-social/indigo/api/bsky"
 	toolsozone "github.com/bluesky-social/indigo/api/ozone"
 	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/bluesky-social/indigo/xrpc"
 )
+
+var newAccountRetryDuration = 3 * 1000 * time.Millisecond
 
 // Helper to hydrate metadata about an account from several sources: PDS (if access), mod service (if access), public identity resolution
 func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (*AccountMeta, error) {
···
 
 	// fetch account metadata from AppView
 	pv, err := appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String())
+	// most common cause of this is a race between automod and ozone/appview for new accounts. just sleep a couple seconds and retry!
+	var xrpcError *xrpc.Error
+	if err != nil && errors.As(err, &xrpcError) && (xrpcError.StatusCode == 400 || xrpcError.StatusCode == 404) {
+		logger.Info("account profile lookup initially failed (from bsky appview), will retry", "err", err, "sleepDuration", newAccountRetryDuration)
+		time.Sleep(newAccountRetryDuration)
+		pv, err = appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String())
+	}
 	if err != nil {
-		logger.Warn("account profile lookup failed", "err", err)
+		logger.Warn("account profile lookup failed (from bsky appview)", "err", err)
 		return &am, nil
 	}
 
···
 			if rd.EmailConfirmedAt != nil && *rd.EmailConfirmedAt != "" {
 				ap.EmailConfirmed = true
 			}
-			ts, err := syntax.ParseDatetimeTime(rd.IndexedAt)
-			if err != nil {
-				return nil, fmt.Errorf("bad account IndexedAt: %w", err)
-			}
-			ap.IndexedAt = ts
+			// TODO: ozone doesn't really return good account "created at", just just leave that field nil
+			ap.IndexedAt = nil
 			if rd.DeactivatedAt != nil {
 				am.Deactivated = true
 			}
···
 			}
 			am.Private = &ap
 		}
-	} else if e.AdminClient != nil {
-		// fall back to PDS/entryway fetching; less metadata available
+	}
+	// fall back to PDS/entryway fetching; less metadata available
+	if am.Private == nil && e.AdminClient != nil {
 		pv, err := comatproto.AdminGetAccountInfo(ctx, e.AdminClient, ident.DID.String())
 		if err != nil {
 			logger.Warn("failed to fetch private account metadata from PDS/entryway", "err", err)
···
 			}
 			ts, err := syntax.ParseDatetimeTime(pv.IndexedAt)
 			if err != nil {
-				return nil, fmt.Errorf("bad account IndexedAt: %w", err)
+				return nil, fmt.Errorf("bad entryway account IndexedAt: %w", err)
 			}
-			ap.IndexedAt = ts
+			ap.IndexedAt = &ts
 			am.Private = &ap
+			if am.CreatedAt == nil {
+				am.CreatedAt = &ts
+			}
 		}
+	}
+
+	if am.CreatedAt == nil {
+		logger.Warn("account metadata missing CreatedAt time")
 	}
 
 	val, err := json.Marshal(&am)

automod/engine/persist.go

···
 		rv, err := toolsozone.ModerationGetRecord(ctx, eng.OzoneClient, c.RecordOp.CID.String(), c.RecordOp.ATURI().String())
 		if err != nil {
 			// NOTE: there is a frequent 4xx error here from Ozone because this record has not been indexed yet
-			c.Logger.Warn("failed to fetch private record metadata", "err", err)
+			c.Logger.Warn("failed to fetch private record metadata from Ozone", "err", err)
 		} else {
 			var existingLabels []string
 			var negLabels []string

automod/pkg.go

···
 
 type Engine = engine.Engine
 type AccountMeta = engine.AccountMeta
+type ProfileSummary = engine.ProfileSummary
+type AccountPrivate = engine.AccountPrivate
 type RuleSet = engine.RuleSet
 
 type Notifier = engine.Notifier

automod/rules/harassment.go

···
 
 // looks for new accounts, which interact with frequently-harassed accounts, and report them for review
 func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
-	if c.Account.Private == nil || c.Account.Identity == nil {
-		return nil
-	}
-	// TODO: helper for account age; and use public info for this (not private)
-	age := time.Since(c.Account.Private.IndexedAt)
-	if age > 7*24*time.Hour {
+	if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) {
 		return nil
 	}
 
···
 		}
 		interactionDIDs = append(interactionDIDs, parentURI.Authority().String())
 	}
-	// TODO: quote-posts; any other interactions?
+	// quote posts
+	if post.Embed != nil && post.Embed.EmbedRecord != nil && post.Embed.EmbedRecord.Record != nil {
+		uri, err := syntax.ParseATURI(post.Embed.EmbedRecord.Record.Uri)
+		if err != nil {
+			c.Logger.Warn("invalid AT-URI in post embed record (quote-post)", "uri", post.Embed.EmbedRecord.Record.Uri)
+		} else {
+			interactionDIDs = append(interactionDIDs, uri.Authority().String())
+		}
+	}
 	if len(interactionDIDs) == 0 {
 		return nil
 	}
 
+	// more than a handful of followers or posts from author account? skip
+	if c.Account.FollowersCount > 10 || c.Account.PostsCount > 10 {
+		return nil
+	}
+	postCount := c.GetCount("post", c.Account.Identity.DID.String(), countstore.PeriodTotal)
+	if postCount > 20 {
+		return nil
+	}
+
 	interactionDIDs = dedupeStrings(interactionDIDs)
 	for _, d := range interactionDIDs {
 		did, err := syntax.ParseDID(d)
···
 		}
 
 		//c.AddRecordFlag("interaction-harassed-target")
+		var privCreatedAt *time.Time
+		if c.Account.Private != nil && c.Account.Private.IndexedAt != nil {
+			privCreatedAt = c.Account.Private.IndexedAt
+		}
+		c.Logger.Warn("possible harassment", "targetDID", did, "author", c.Account.Identity.DID, "accountCreated", c.Account.CreatedAt, "privateAccountCreated", privCreatedAt)
 		c.ReportAccount(automod.ReportReasonOther, fmt.Sprintf("possible harassment of known target account: %s (also labeled; remove label if this isn't harassment)", did))
 		c.AddAccountLabel("!hide")
 		c.Notify("slack")
···
 
 // looks for new accounts, which frequently post the same type of content
 func HarassmentTrivialPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
-	if c.Account.Private == nil || c.Account.Identity == nil {
-		return nil
-	}
-	// TODO: helper for account age; and use public info for this (not private)
-	age := time.Since(c.Account.Private.IndexedAt)
-	if age > 7*24*time.Hour {
+	if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) {
 		return nil
 	}
 

automod/rules/helpers.go

···
 import (
 	"fmt"
 	"regexp"
+	"time"
 
 	appbsky "github.com/bluesky-social/indigo/api/bsky"
 	"github.com/bluesky-social/indigo/atproto/syntax"
···
 	}
 	return false
 }
+
+// no accounts exist before this time
+var atprotoAccountEpoch = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
+
+// returns true if account creation timestamp is plausible: not-nil, not in distant past, not in the future
+func plausibleAccountCreation(when *time.Time) bool {
+	if when == nil {
+		return false
+	}
+	// this is mostly to check for misconfigurations or null values (eg, UNIX epoch zero means "unknown" not actually 1970)
+	if !when.After(atprotoAccountEpoch) {
+		return false
+	}
+	// a timestamp in the future would also indicate some misconfiguration
+	if when.After(time.Now().Add(time.Hour)) {
+		return false
+	}
+	return true
+}
+
+// checks if account was created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false'
+func AccountIsYoungerThan(c *automod.AccountContext, age time.Duration) bool {
+	// TODO: consider swapping priority order here (and below)
+	if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) {
+		return time.Since(*c.Account.CreatedAt) < age
+	}
+	if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) {
+		return time.Since(*c.Account.Private.IndexedAt) < age
+	}
+	return false
+}
+
+// checks if account was *not* created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false'
+func AccountIsOlderThan(c *automod.AccountContext, age time.Duration) bool {
+	if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) {
+		return time.Since(*c.Account.CreatedAt) >= age
+	}
+	if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) {
+		return time.Since(*c.Account.Private.IndexedAt) >= age
+	}
+	return false
+}

automod/rules/helpers_test.go

···
 
 import (
 	"testing"
+	"time"
 
+	"github.com/bluesky-social/indigo/atproto/identity"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/bluesky-social/indigo/automod"
 	"github.com/bluesky-social/indigo/automod/keyword"
 	"github.com/stretchr/testify/assert"
 )
···
 	// hashing function should be consistent over time
 	assert.Equal("4e6f69c0e3d10992", HashOfString("dummy-value"))
 }
+
+func TestAccountIsYoungerThan(t *testing.T) {
+	assert := assert.New(t)
+
+	am := automod.AccountMeta{
+		Identity: &identity.Identity{
+			DID:    syntax.DID("did:plc:abc111"),
+			Handle: syntax.Handle("handle.example.com"),
+		},
+		Profile: automod.ProfileSummary{},
+		Private: nil,
+	}
+	now := time.Now()
+	ac := automod.AccountContext{
+		Account: am,
+	}
+	assert.False(AccountIsYoungerThan(&ac, time.Hour))
+	assert.False(AccountIsOlderThan(&ac, time.Hour))
+
+	ac.Account.CreatedAt = &now
+	assert.True(AccountIsYoungerThan(&ac, time.Hour))
+	assert.False(AccountIsOlderThan(&ac, time.Hour))
+
+	yesterday := time.Now().Add(-1 * time.Hour * 24)
+	ac.Account.CreatedAt = &yesterday
+	assert.False(AccountIsYoungerThan(&ac, time.Hour))
+	assert.True(AccountIsOlderThan(&ac, time.Hour))
+
+	old := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)
+	ac.Account.CreatedAt = &old
+	assert.False(AccountIsYoungerThan(&ac, time.Hour))
+	assert.False(AccountIsYoungerThan(&ac, time.Hour*24*365*100))
+	assert.False(AccountIsOlderThan(&ac, time.Hour))
+	assert.False(AccountIsOlderThan(&ac, time.Hour*24*365*100))
+
+	future := time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC)
+	ac.Account.CreatedAt = &future
+	assert.False(AccountIsYoungerThan(&ac, time.Hour))
+	assert.False(AccountIsOlderThan(&ac, time.Hour))
+
+	ac.Account.CreatedAt = nil
+	ac.Account.Private = &automod.AccountPrivate{
+		Email:     "[e-mail]",
+		IndexedAt: &yesterday,
+	}
+	assert.True(AccountIsYoungerThan(&ac, 48*time.Hour))
+	assert.False(AccountIsYoungerThan(&ac, time.Hour))
+	assert.True(AccountIsOlderThan(&ac, time.Hour))
+	assert.False(AccountIsOlderThan(&ac, 48*time.Hour))
+}

automod/rules/identity.go

···
 
 // triggers on first identity event for an account (DID)
 func NewAccountRule(c *automod.AccountContext) error {
-	// need access to IndexedAt for this rule
-	if c.Account.Private == nil || c.Account.Identity == nil {
+	if c.Account.Identity == nil || !AccountIsYoungerThan(c, 4*time.Hour) {
 		return nil
 	}
 
 	did := c.Account.Identity.DID.String()
-	age := time.Since(c.Account.Private.IndexedAt)
-	if age > 4*time.Hour {
-		return nil
-	}
 	exists := c.GetCount("acct/exists", did, countstore.PeriodTotal)
 	if exists == 0 {
 		c.Logger.Info("new account")

automod/rules/mentions.go

···
 var _ automod.PostRuleFunc = YoungAccountDistinctMentionsRule
 
 func YoungAccountDistinctMentionsRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
-
-	// only young posting accounts
-	if c.Account.Private != nil {
-		age := time.Since(c.Account.Private.IndexedAt)
-		if age > 2*7*24*time.Hour {
-			return nil
-		}
+	if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) {
+		return nil
 	}
 
 	// parse out all the mentions

automod/rules/nostr.go

···
 
 // looks for new accounts, which frequently post the same type of content
 func NostrSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
-	if c.Account.Identity == nil {
+	if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) {
 		return nil
-	}
-
-	// often don't have private metadata for these accounts right after creation
-	if c.Account.Private != nil {
-		// TODO: helper for account age; and use public info for this (not private)
-		age := time.Since(c.Account.Private.IndexedAt)
-		if age > 2*24*time.Hour {
-			return nil
-		}
 	}
 
 	// is this a bridged nostr account? if not, bail out

automod/rules/promo.go

···
 //
 // this rule depends on ReplyCountPostRule() to set counts
 func AggressivePromotionRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
-	if c.Account.Private == nil || c.Account.Identity == nil {
-		return nil
-	}
-	// TODO: helper for account age
-	age := time.Since(c.Account.Private.IndexedAt)
-	if age > 7*24*time.Hour {
+	if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) {
 		return nil
 	}
 	if post.Reply == nil || IsSelfThread(c, post) {

automod/rules/quick.go

···
 var _ automod.IdentityRuleFunc = NewAccountBotEmailRule
 
 func NewAccountBotEmailRule(c *automod.AccountContext) error {
-	// need access to IndexedAt for this rule
-	if c.Account.Private == nil || c.Account.Identity == nil {
-		return nil
-	}
-
-	age := time.Since(c.Account.Private.IndexedAt)
-	if age > 1*time.Hour {
+	if c.Account.Identity == nil || !AccountIsYoungerThan(c, 1*time.Hour) {
 		return nil
 	}
 

automod/rules/replies.go

···
 	if utf8.RuneCountInString(post.Text) <= 10 {
 		return nil
 	}
-	if c.Account.Private != nil {
-		age := time.Since(c.Account.Private.IndexedAt)
-		if age > 2*7*24*time.Hour {
-			return nil
-		}
+	if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) {
+		return nil
 	}
 
 	// don't count if there is a follow-back relationship
···
 	if utf8.RuneCountInString(post.Text) <= 10 {
 		return nil
 	}
-	if c.Account.Private != nil {
-		age := time.Since(c.Account.Private.IndexedAt)
-		if age > 2*7*24*time.Hour {
-			return nil
-		}
+	if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) {
+		return nil
 	}
 
 	// don't count if there is a follow-back relationship

automod/rules/reposts.go

···
 
 // looks for accounts which do frequent reposts
 func TooManyRepostRule(c *automod.RecordContext) error {
+	// Don't bother checking reposts from accounts older than 30 days
+	if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) {
+		return nil
+	}
 
 	did := c.Account.Identity.DID.String()
-	// Don't bother checking reposts from accounts older than 30 days
-	if c.Account.Private != nil {
-		age := time.Since(c.Account.Private.IndexedAt)
-		if age > 30*24*time.Hour {
-			return nil
-		}
-	}
 
 	// Special case for newsmast bridge feeds
 	handle := c.Account.Identity.Handle.String()

cmd/hepa/server.go

···
 		}
 		counters = cnt
 
-		csh, err := cachestore.NewRedisCacheStore(config.RedisURL, 30*time.Minute)
+		csh, err := cachestore.NewRedisCacheStore(config.RedisURL, 6*time.Hour)
 		if err != nil {
 			return nil, fmt.Errorf("initializing redis cachestore: %v", err)
 		}
···
 		flags = flg
 	} else {
 		counters = countstore.NewMemCountStore()
-		cache = cachestore.NewMemCacheStore(5_000, 30*time.Minute)
+		cache = cachestore.NewMemCacheStore(5_000, 1*time.Hour)
 		flags = flagstore.NewMemFlagStore()
 	}