+20
-5
internal/api/handlers.go
+20
-5
internal/api/handlers.go
···
276
276
if pds.IPInfo.ASN > 0 {
277
277
response["asn"] = pds.IPInfo.ASN
278
278
}
279
-
if pds.IPInfo.IsDatacenter {
280
-
response["is_datacenter"] = pds.IPInfo.IsDatacenter
281
-
}
279
+
280
+
// Add all network type flags
281
+
response["is_datacenter"] = pds.IPInfo.IsDatacenter
282
+
response["is_vpn"] = pds.IPInfo.IsVPN
283
+
response["is_crawler"] = pds.IPInfo.IsCrawler
284
+
response["is_tor"] = pds.IPInfo.IsTor
285
+
response["is_proxy"] = pds.IPInfo.IsProxy
286
+
287
+
// Add computed is_home field
288
+
response["is_home"] = pds.IPInfo.IsHome()
282
289
}
283
290
284
291
return response
···
316
323
}
317
324
}
318
325
319
-
// Add full IP info
326
+
// Add full IP info with computed is_home field
320
327
if pds.IPInfo != nil {
321
-
response["ip_info"] = pds.IPInfo
328
+
// Convert IPInfo to map
329
+
ipInfoMap := make(map[string]interface{})
330
+
ipInfoJSON, _ := json.Marshal(pds.IPInfo)
331
+
json.Unmarshal(ipInfoJSON, &ipInfoMap)
332
+
333
+
// Add computed is_home field
334
+
ipInfoMap["is_home"] = pds.IPInfo.IsHome()
335
+
336
+
response["ip_info"] = ipInfoMap
322
337
}
323
338
324
339
return response
+36
-13
internal/ipinfo/client.go
+36
-13
internal/ipinfo/client.go
···
99
99
return ipInfo, nil
100
100
}
101
101
102
-
// ExtractIPFromEndpoint extracts IP from endpoint URL
103
-
func ExtractIPFromEndpoint(endpoint string) (string, error) {
102
+
// IPAddresses holds both IPv4 and IPv6 addresses
103
+
type IPAddresses struct {
104
+
IPv4 string
105
+
IPv6 string
106
+
}
107
+
108
+
// ExtractIPsFromEndpoint extracts both IPv4 and IPv6 from endpoint URL
109
+
func ExtractIPsFromEndpoint(endpoint string) (*IPAddresses, error) {
104
110
// Parse URL
105
111
parsedURL, err := url.Parse(endpoint)
106
112
if err != nil {
107
-
return "", fmt.Errorf("failed to parse endpoint URL: %w", err)
113
+
return nil, fmt.Errorf("failed to parse endpoint URL: %w", err)
108
114
}
109
115
110
116
host := parsedURL.Hostname()
111
117
if host == "" {
112
-
return "", fmt.Errorf("no hostname in endpoint")
118
+
return nil, fmt.Errorf("no hostname in endpoint")
113
119
}
120
+
121
+
result := &IPAddresses{}
114
122
115
123
// Check if host is already an IP
116
-
if net.ParseIP(host) != nil {
117
-
return host, nil
124
+
if ip := net.ParseIP(host); ip != nil {
125
+
if ip.To4() != nil {
126
+
result.IPv4 = host
127
+
} else {
128
+
result.IPv6 = host
129
+
}
130
+
return result, nil
118
131
}
119
132
120
-
// Resolve hostname to IP
133
+
// Resolve hostname to IPs
121
134
ips, err := net.LookupIP(host)
122
135
if err != nil {
123
-
return "", fmt.Errorf("failed to resolve hostname: %w", err)
136
+
return nil, fmt.Errorf("failed to resolve hostname: %w", err)
124
137
}
125
138
126
139
if len(ips) == 0 {
127
-
return "", fmt.Errorf("no IPs found for hostname")
140
+
return nil, fmt.Errorf("no IPs found for hostname")
128
141
}
129
142
130
-
// Return first IPv4 address
143
+
// Extract both IPv4 and IPv6
131
144
for _, ip := range ips {
132
145
if ipv4 := ip.To4(); ipv4 != nil {
133
-
return ipv4.String(), nil
146
+
if result.IPv4 == "" {
147
+
result.IPv4 = ipv4.String()
148
+
}
149
+
} else {
150
+
if result.IPv6 == "" {
151
+
result.IPv6 = ip.String()
152
+
}
134
153
}
135
154
}
136
155
137
-
// Fallback to first IP (might be IPv6)
138
-
return ips[0].String(), nil
156
+
// Must have at least one IP
157
+
if result.IPv4 == "" && result.IPv6 == "" {
158
+
return nil, fmt.Errorf("no valid IPs found")
159
+
}
160
+
161
+
return result, nil
139
162
}
+12
-7
internal/pds/scanner.go
+12
-7
internal/pds/scanner.go
···
124
124
}
125
125
126
126
func (s *Scanner) scanAndSaveEndpoint(ctx context.Context, ep *storage.Endpoint) {
127
-
// STEP 1: Resolve IP (before any network call)
128
-
ip, err := ipinfo.ExtractIPFromEndpoint(ep.Endpoint)
127
+
// STEP 1: Resolve IPs (both IPv4 and IPv6)
128
+
ips, err := ipinfo.ExtractIPsFromEndpoint(ep.Endpoint)
129
129
if err != nil {
130
130
// Mark as offline due to DNS failure
131
131
s.saveScanResult(ctx, ep.ID, &ScanResult{
···
135
135
return
136
136
}
137
137
138
-
// Update IP immediately
139
-
s.db.UpdateEndpointIP(ctx, ep.ID, ip, time.Now().UTC())
138
+
// Update IPs immediately
139
+
s.db.UpdateEndpointIPs(ctx, ep.ID, ips.IPv4, ips.IPv6, time.Now().UTC())
140
140
141
-
// STEP 1.5: Fetch IP info asynchronously ASAP (runs in parallel with scanning)
142
-
go s.updateIPInfoIfNeeded(ctx, ip)
141
+
// STEP 1.5: Fetch IP info asynchronously for both IPs
142
+
if ips.IPv4 != "" {
143
+
go s.updateIPInfoIfNeeded(ctx, ips.IPv4)
144
+
}
145
+
if ips.IPv6 != "" {
146
+
go s.updateIPInfoIfNeeded(ctx, ips.IPv6)
147
+
}
143
148
144
-
// STEP 2: Health check
149
+
// STEP 2: Health check (rest remains the same)
145
150
available, responseTime, version, err := s.client.CheckHealth(ctx, ep.Endpoint)
146
151
if err != nil || !available {
147
152
errMsg := "health check failed"
+1
-1
internal/storage/db.go
+1
-1
internal/storage/db.go
···
27
27
EndpointExists(ctx context.Context, endpoint string, endpointType string) (bool, error)
28
28
GetEndpointIDByEndpoint(ctx context.Context, endpoint string, endpointType string) (int64, error)
29
29
GetEndpointScans(ctx context.Context, endpointID int64, limit int) ([]*EndpointScan, error)
30
-
UpdateEndpointIP(ctx context.Context, endpointID int64, ip string, resolvedAt time.Time) error
30
+
UpdateEndpointIPs(ctx context.Context, endpointID int64, ipv4, ipv6 string, resolvedAt time.Time) error
31
31
SaveEndpointScan(ctx context.Context, scan *EndpointScan) error
32
32
SetScanRetention(retention int)
33
33
UpdateEndpointStatus(ctx context.Context, endpointID int64, update *EndpointUpdate) error
+111
-91
internal/storage/postgres.go
+111
-91
internal/storage/postgres.go
···
73
73
log.Info("Running database migrations...")
74
74
75
75
schema := `
76
-
-- Endpoints table (NO user_count, NO ip_info)
77
-
CREATE TABLE IF NOT EXISTS endpoints (
78
-
id BIGSERIAL PRIMARY KEY,
79
-
endpoint_type TEXT NOT NULL DEFAULT 'pds',
80
-
endpoint TEXT NOT NULL,
81
-
server_did TEXT,
82
-
discovered_at TIMESTAMP NOT NULL,
83
-
last_checked TIMESTAMP,
84
-
status INTEGER DEFAULT 0,
85
-
ip TEXT,
86
-
ip_resolved_at TIMESTAMP,
87
-
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
88
-
UNIQUE(endpoint_type, endpoint)
89
-
);
76
+
-- Endpoints table (NO user_count, NO ip_info)
77
+
CREATE TABLE IF NOT EXISTS endpoints (
78
+
id BIGSERIAL PRIMARY KEY,
79
+
endpoint_type TEXT NOT NULL DEFAULT 'pds',
80
+
endpoint TEXT NOT NULL,
81
+
server_did TEXT,
82
+
discovered_at TIMESTAMP NOT NULL,
83
+
last_checked TIMESTAMP,
84
+
status INTEGER DEFAULT 0,
85
+
ip TEXT,
86
+
ipv6 TEXT,
87
+
ip_resolved_at TIMESTAMP,
88
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
89
+
UNIQUE(endpoint_type, endpoint)
90
+
);
90
91
91
-
CREATE INDEX IF NOT EXISTS idx_endpoints_type_endpoint ON endpoints(endpoint_type, endpoint);
92
-
CREATE INDEX IF NOT EXISTS idx_endpoints_status ON endpoints(status);
93
-
CREATE INDEX IF NOT EXISTS idx_endpoints_type ON endpoints(endpoint_type);
94
-
CREATE INDEX IF NOT EXISTS idx_endpoints_ip ON endpoints(ip);
95
-
CREATE INDEX IF NOT EXISTS idx_endpoints_server_did ON endpoints(server_did);
92
+
CREATE INDEX IF NOT EXISTS idx_endpoints_type_endpoint ON endpoints(endpoint_type, endpoint);
93
+
CREATE INDEX IF NOT EXISTS idx_endpoints_status ON endpoints(status);
94
+
CREATE INDEX IF NOT EXISTS idx_endpoints_type ON endpoints(endpoint_type);
95
+
CREATE INDEX IF NOT EXISTS idx_endpoints_ip ON endpoints(ip);
96
+
CREATE INDEX IF NOT EXISTS idx_endpoints_ipv6 ON endpoints(ipv6);
97
+
CREATE INDEX IF NOT EXISTS idx_endpoints_server_did ON endpoints(server_did);
96
98
CREATE INDEX IF NOT EXISTS idx_endpoints_server_did_type_discovered ON endpoints(server_did, endpoint_type, discovered_at);
97
99
98
-
-- IP infos table (IP as PRIMARY KEY)
99
-
CREATE TABLE IF NOT EXISTS ip_infos (
100
-
ip TEXT PRIMARY KEY,
101
-
city TEXT,
102
-
country TEXT,
103
-
country_code TEXT,
104
-
asn INTEGER,
105
-
asn_org TEXT,
106
-
is_datacenter BOOLEAN,
107
-
is_vpn BOOLEAN,
108
-
latitude REAL,
109
-
longitude REAL,
110
-
raw_data JSONB,
111
-
fetched_at TIMESTAMP NOT NULL,
112
-
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
113
-
);
100
+
-- IP infos table (IP as PRIMARY KEY)
101
+
CREATE TABLE IF NOT EXISTS ip_infos (
102
+
ip TEXT PRIMARY KEY,
103
+
city TEXT,
104
+
country TEXT,
105
+
country_code TEXT,
106
+
asn INTEGER,
107
+
asn_org TEXT,
108
+
is_datacenter BOOLEAN,
109
+
is_vpn BOOLEAN,
110
+
is_crawler BOOLEAN,
111
+
is_tor BOOLEAN,
112
+
is_proxy BOOLEAN,
113
+
latitude REAL,
114
+
longitude REAL,
115
+
raw_data JSONB,
116
+
fetched_at TIMESTAMP NOT NULL,
117
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
118
+
);
114
119
115
-
CREATE INDEX IF NOT EXISTS idx_ip_infos_country_code ON ip_infos(country_code);
116
-
CREATE INDEX IF NOT EXISTS idx_ip_infos_asn ON ip_infos(asn);
120
+
CREATE INDEX IF NOT EXISTS idx_ip_infos_country_code ON ip_infos(country_code);
121
+
CREATE INDEX IF NOT EXISTS idx_ip_infos_asn ON ip_infos(asn);
117
122
118
123
-- Endpoint scans (renamed from pds_scans)
119
124
CREATE TABLE IF NOT EXISTS endpoint_scans (
···
237
242
238
243
func (p *PostgresDB) UpsertEndpoint(ctx context.Context, endpoint *Endpoint) error {
239
244
query := `
240
-
INSERT INTO endpoints (endpoint_type, endpoint, discovered_at, last_checked, status, ip, ip_resolved_at)
241
-
VALUES ($1, $2, $3, $4, $5, $6, $7)
245
+
INSERT INTO endpoints (endpoint_type, endpoint, discovered_at, last_checked, status, ip, ipv6, ip_resolved_at)
246
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
242
247
ON CONFLICT(endpoint_type, endpoint) DO UPDATE SET
243
248
last_checked = EXCLUDED.last_checked,
244
249
status = EXCLUDED.status,
···
246
251
WHEN EXCLUDED.ip IS NOT NULL AND EXCLUDED.ip != '' THEN EXCLUDED.ip
247
252
ELSE endpoints.ip
248
253
END,
254
+
ipv6 = CASE
255
+
WHEN EXCLUDED.ipv6 IS NOT NULL AND EXCLUDED.ipv6 != '' THEN EXCLUDED.ipv6
256
+
ELSE endpoints.ipv6
257
+
END,
249
258
ip_resolved_at = CASE
250
-
WHEN EXCLUDED.ip IS NOT NULL AND EXCLUDED.ip != '' THEN EXCLUDED.ip_resolved_at
259
+
WHEN (EXCLUDED.ip IS NOT NULL AND EXCLUDED.ip != '') OR (EXCLUDED.ipv6 IS NOT NULL AND EXCLUDED.ipv6 != '') THEN EXCLUDED.ip_resolved_at
251
260
ELSE endpoints.ip_resolved_at
252
261
END,
253
262
updated_at = CURRENT_TIMESTAMP
···
255
264
`
256
265
err := p.db.QueryRowContext(ctx, query,
257
266
endpoint.EndpointType, endpoint.Endpoint, endpoint.DiscoveredAt,
258
-
endpoint.LastChecked, endpoint.Status, endpoint.IP, endpoint.IPResolvedAt).Scan(&endpoint.ID)
267
+
endpoint.LastChecked, endpoint.Status, endpoint.IP, endpoint.IPv6, endpoint.IPResolvedAt).Scan(&endpoint.ID)
259
268
return err
260
269
}
261
270
···
276
285
func (p *PostgresDB) GetEndpoint(ctx context.Context, endpoint string, endpointType string) (*Endpoint, error) {
277
286
query := `
278
287
SELECT id, endpoint_type, endpoint, discovered_at, last_checked, status,
279
-
ip, ip_resolved_at, updated_at
288
+
ip, ipv6, ip_resolved_at, updated_at
280
289
FROM endpoints
281
290
WHERE endpoint = $1 AND endpoint_type = $2
282
291
`
283
292
284
293
var ep Endpoint
285
294
var lastChecked, ipResolvedAt sql.NullTime
286
-
var ip sql.NullString
295
+
var ip, ipv6 sql.NullString
287
296
288
297
err := p.db.QueryRowContext(ctx, query, endpoint, endpointType).Scan(
289
298
&ep.ID, &ep.EndpointType, &ep.Endpoint, &ep.DiscoveredAt, &lastChecked,
290
-
&ep.Status, &ip, &ipResolvedAt, &ep.UpdatedAt,
299
+
&ep.Status, &ip, &ipv6, &ipResolvedAt, &ep.UpdatedAt,
291
300
)
292
301
if err != nil {
293
302
return nil, err
···
299
308
if ip.Valid {
300
309
ep.IP = ip.String
301
310
}
311
+
if ipv6.Valid {
312
+
ep.IPv6 = ipv6.String
313
+
}
302
314
if ipResolvedAt.Valid {
303
315
ep.IPResolvedAt = ipResolvedAt.Time
304
316
}
···
308
320
309
321
func (p *PostgresDB) GetEndpoints(ctx context.Context, filter *EndpointFilter) ([]*Endpoint, error) {
310
322
query := `
311
-
SELECT DISTINCT ON (COALESCE(server_did, id::text))
312
-
id, endpoint_type, endpoint, server_did, discovered_at, last_checked, status,
313
-
ip, ip_resolved_at, updated_at
314
-
FROM endpoints
315
-
WHERE 1=1
323
+
SELECT DISTINCT ON (COALESCE(server_did, id::text))
324
+
id, endpoint_type, endpoint, server_did, discovered_at, last_checked, status,
325
+
ip, ipv6, ip_resolved_at, updated_at
326
+
FROM endpoints
327
+
WHERE 1=1
316
328
`
317
329
args := []interface{}{}
318
330
argIdx := 1
···
363
375
for rows.Next() {
364
376
var ep Endpoint
365
377
var lastChecked, ipResolvedAt sql.NullTime
366
-
var ip, serverDID sql.NullString
378
+
var ip, ipv6, serverDID sql.NullString
367
379
368
380
err := rows.Scan(
369
381
&ep.ID, &ep.EndpointType, &ep.Endpoint, &serverDID, &ep.DiscoveredAt, &lastChecked,
370
-
&ep.Status, &ip, &ipResolvedAt, &ep.UpdatedAt,
382
+
&ep.Status, &ip, &ipv6, &ipResolvedAt, &ep.UpdatedAt,
371
383
)
372
384
if err != nil {
373
385
return nil, err
···
381
393
}
382
394
if ip.Valid {
383
395
ep.IP = ip.String
396
+
}
397
+
if ipv6.Valid {
398
+
ep.IPv6 = ipv6.String
384
399
}
385
400
if ipResolvedAt.Valid {
386
401
ep.IPResolvedAt = ipResolvedAt.Time
···
402
417
return err
403
418
}
404
419
405
-
func (p *PostgresDB) UpdateEndpointIP(ctx context.Context, endpointID int64, ip string, resolvedAt time.Time) error {
420
+
func (p *PostgresDB) UpdateEndpointIPs(ctx context.Context, endpointID int64, ipv4, ipv6 string, resolvedAt time.Time) error {
406
421
query := `
407
422
UPDATE endpoints
408
-
SET ip = $1, ip_resolved_at = $2, updated_at = $3
409
-
WHERE id = $4
423
+
SET ip = $1, ipv6 = $2, ip_resolved_at = $3, updated_at = $4
424
+
WHERE id = $5
410
425
`
411
-
_, err := p.db.ExecContext(ctx, query, ip, resolvedAt, time.Now().UTC(), endpointID)
426
+
_, err := p.db.ExecContext(ctx, query, ipv4, ipv6, resolvedAt, time.Now().UTC(), endpointID)
412
427
return err
413
428
}
414
429
···
577
592
e.id, e.endpoint, e.server_did, e.discovered_at, e.last_checked, e.status, e.ip,
578
593
latest.user_count, latest.response_time, latest.version, latest.scanned_at,
579
594
i.city, i.country, i.country_code, i.asn, i.asn_org,
580
-
i.is_datacenter, i.is_vpn, i.latitude, i.longitude
595
+
i.is_datacenter, i.is_vpn, i.is_crawler, i.is_tor, i.is_proxy,
596
+
i.latitude, i.longitude
581
597
FROM unique_servers e
582
598
LEFT JOIN LATERAL (
583
599
SELECT
···
636
652
item := &PDSListItem{}
637
653
var ip, serverDID, city, country, countryCode, asnOrg sql.NullString
638
654
var asn sql.NullInt32
639
-
var isDatacenter, isVPN sql.NullBool
655
+
var isDatacenter, isVPN, isCrawler, isTor, isProxy sql.NullBool
640
656
var lat, lon sql.NullFloat64
641
657
var userCount sql.NullInt32
642
658
var responseTime sql.NullFloat64
···
647
663
&item.ID, &item.Endpoint, &serverDID, &item.DiscoveredAt, &item.LastChecked, &item.Status, &ip,
648
664
&userCount, &responseTime, &version, &scannedAt,
649
665
&city, &country, &countryCode, &asn, &asnOrg,
650
-
&isDatacenter, &isVPN, &lat, &lon,
666
+
&isDatacenter, &isVPN, &isCrawler, &isTor, &isProxy,
667
+
&lat, &lon,
651
668
)
652
669
if err != nil {
653
670
return nil, err
···
686
703
ASNOrg: asnOrg.String,
687
704
IsDatacenter: isDatacenter.Bool,
688
705
IsVPN: isVPN.Bool,
706
+
IsCrawler: isCrawler.Bool,
707
+
IsTor: isTor.Bool,
708
+
IsProxy: isProxy.Bool,
689
709
Latitude: float32(lat.Float64),
690
710
Longitude: float32(lon.Float64),
691
711
}
···
736
756
latest.scan_data->'metadata'->'server_info' as server_info,
737
757
latest.scanned_at,
738
758
i.city, i.country, i.country_code, i.asn, i.asn_org,
739
-
i.is_datacenter, i.is_vpn, i.latitude, i.longitude,
759
+
i.is_datacenter, i.is_vpn, i.is_crawler, i.is_tor, i.is_proxy,
760
+
i.latitude, i.longitude,
740
761
i.raw_data,
741
762
COALESCE(aa.aliases, ARRAY[]::text[]) as aliases,
742
763
aa.first_discovered_at
···
755
776
detail := &PDSDetail{}
756
777
var ip, city, country, countryCode, asnOrg, serverDID sql.NullString
757
778
var asn sql.NullInt32
758
-
var isDatacenter, isVPN sql.NullBool
779
+
var isDatacenter, isVPN, isCrawler, isTor, isProxy sql.NullBool
759
780
var lat, lon sql.NullFloat64
760
781
var userCount sql.NullInt32
761
782
var responseTime sql.NullFloat64
···
770
791
&detail.ID, &detail.Endpoint, &serverDID, &detail.DiscoveredAt, &detail.LastChecked, &detail.Status, &ip,
771
792
&userCount, &responseTime, &version, &serverInfoJSON, &scannedAt,
772
793
&city, &country, &countryCode, &asn, &asnOrg,
773
-
&isDatacenter, &isVPN, &lat, &lon,
794
+
&isDatacenter, &isVPN, &isCrawler, &isTor, &isProxy,
795
+
&lat, &lon,
774
796
&rawDataJSON,
775
797
pq.Array(&aliases),
776
798
&firstDiscoveredAt,
···
820
842
}
821
843
}
822
844
823
-
// Parse IP info
845
+
// Parse IP info with all fields
824
846
if city.Valid || country.Valid {
825
847
detail.IPInfo = &IPInfo{
826
848
IP: ip.String,
···
831
853
ASNOrg: asnOrg.String,
832
854
IsDatacenter: isDatacenter.Bool,
833
855
IsVPN: isVPN.Bool,
856
+
IsCrawler: isCrawler.Bool,
857
+
IsTor: isTor.Bool,
858
+
IsProxy: isProxy.Bool,
834
859
Latitude: float32(lat.Float64),
835
860
Longitude: float32(lon.Float64),
836
861
}
···
978
1003
isVPN = val
979
1004
}
980
1005
1006
+
isCrawler := false
1007
+
if val, ok := ipInfo["is_crawler"].(bool); ok {
1008
+
isCrawler = val
1009
+
}
1010
+
1011
+
isTor := false
1012
+
if val, ok := ipInfo["is_tor"].(bool); ok {
1013
+
isTor = val
1014
+
}
1015
+
1016
+
isProxy := false
1017
+
if val, ok := ipInfo["is_proxy"].(bool); ok {
1018
+
isProxy = val
1019
+
}
1020
+
981
1021
lat := extractFloat(ipInfo, "location", "latitude")
982
1022
lon := extractFloat(ipInfo, "location", "longitude")
983
1023
984
1024
query := `
985
-
INSERT INTO ip_infos (ip, city, country, country_code, asn, asn_org, is_datacenter, is_vpn, latitude, longitude, raw_data, fetched_at)
986
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
1025
+
INSERT INTO ip_infos (ip, city, country, country_code, asn, asn_org, is_datacenter, is_vpn, is_crawler, is_tor, is_proxy, latitude, longitude, raw_data, fetched_at)
1026
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
987
1027
ON CONFLICT(ip) DO UPDATE SET
988
1028
city = EXCLUDED.city,
989
1029
country = EXCLUDED.country,
···
992
1032
asn_org = EXCLUDED.asn_org,
993
1033
is_datacenter = EXCLUDED.is_datacenter,
994
1034
is_vpn = EXCLUDED.is_vpn,
1035
+
is_crawler = EXCLUDED.is_crawler,
1036
+
is_tor = EXCLUDED.is_tor,
1037
+
is_proxy = EXCLUDED.is_proxy,
995
1038
latitude = EXCLUDED.latitude,
996
1039
longitude = EXCLUDED.longitude,
997
1040
raw_data = EXCLUDED.raw_data,
998
1041
fetched_at = EXCLUDED.fetched_at,
999
1042
updated_at = CURRENT_TIMESTAMP
1000
1043
`
1001
-
_, err := p.db.ExecContext(ctx, query, ip, city, country, countryCode, asn, asnOrg, isDatacenter, isVPN, lat, lon, rawDataJSON, time.Now().UTC())
1044
+
_, err := p.db.ExecContext(ctx, query, ip, city, country, countryCode, asn, asnOrg, isDatacenter, isVPN, isCrawler, isTor, isProxy, lat, lon, rawDataJSON, time.Now().UTC())
1002
1045
return err
1003
1046
}
1004
1047
1005
1048
func (p *PostgresDB) GetIPInfo(ctx context.Context, ip string) (*IPInfo, error) {
1006
1049
query := `
1007
-
SELECT ip, city, country, country_code, asn, asn_org, is_datacenter, is_vpn,
1050
+
SELECT ip, city, country, country_code, asn, asn_org, is_datacenter, is_vpn, is_crawler, is_tor, is_proxy,
1008
1051
latitude, longitude, raw_data, fetched_at, updated_at
1009
1052
FROM ip_infos
1010
1053
WHERE ip = $1
···
1015
1058
1016
1059
err := p.db.QueryRowContext(ctx, query, ip).Scan(
1017
1060
&info.IP, &info.City, &info.Country, &info.CountryCode, &info.ASN, &info.ASNOrg,
1018
-
&info.IsDatacenter, &info.IsVPN, &info.Latitude, &info.Longitude,
1061
+
&info.IsDatacenter, &info.IsVPN, &info.IsCrawler, &info.IsTor, &info.IsProxy,
1062
+
&info.Latitude, &info.Longitude,
1019
1063
&rawDataJSON, &info.FetchedAt, &info.UpdatedAt,
1020
1064
)
1021
1065
if err != nil {
···
1103
1147
}
1104
1148
}
1105
1149
return 0
1106
-
}
1107
-
1108
-
func extractBool(data map[string]interface{}, keys ...string) bool {
1109
-
current := data
1110
-
for i, key := range keys {
1111
-
if i == len(keys)-1 {
1112
-
if val, ok := current[key].(bool); ok {
1113
-
return val
1114
-
}
1115
-
// Check if it's a string that matches (for type="hosting")
1116
-
if val, ok := current[key].(string); ok {
1117
-
// For cases like company.type == "hosting"
1118
-
expectedValue := keys[len(keys)-1]
1119
-
return val == expectedValue
1120
-
}
1121
-
return false
1122
-
}
1123
-
if nested, ok := current[key].(map[string]interface{}); ok {
1124
-
current = nested
1125
-
} else {
1126
-
return false
1127
-
}
1128
-
}
1129
-
return false
1130
1150
}
1131
1151
1132
1152
// ===== BUNDLE OPERATIONS =====
+10
internal/storage/types.go
+10
internal/storage/types.go
···
26
26
LastChecked time.Time
27
27
Status int
28
28
IP string
29
+
IPv6 string // NEW
29
30
IPResolvedAt time.Time
30
31
UpdatedAt time.Time
31
32
}
···
194
195
ASNOrg string `json:"asn_org,omitempty"`
195
196
IsDatacenter bool `json:"is_datacenter"`
196
197
IsVPN bool `json:"is_vpn"`
198
+
IsCrawler bool `json:"is_crawler"`
199
+
IsTor bool `json:"is_tor"`
200
+
IsProxy bool `json:"is_proxy"`
197
201
Latitude float32 `json:"latitude,omitempty"`
198
202
Longitude float32 `json:"longitude,omitempty"`
199
203
RawData map[string]interface{} `json:"raw_data,omitempty"`
200
204
FetchedAt time.Time `json:"fetched_at"`
201
205
UpdatedAt time.Time `json:"updated_at"`
206
+
}
207
+
208
+
// IsHome returns true if this is a residential/home IP
209
+
// (not crawler, datacenter, tor, proxy, or vpn)
210
+
func (i *IPInfo) IsHome() bool {
211
+
return !i.IsCrawler && !i.IsDatacenter && !i.IsTor && !i.IsProxy && !i.IsVPN
202
212
}
203
213
204
214
// PDSListItem is a virtual type created by JOIN for /pds endpoint