Skip to content

Commit

Permalink
Http Piece Retrival Test (#8)
Browse files Browse the repository at this point in the history
  • Loading branch information
@stephen-pl
stephen-pl authored Oct 30, 2023
1 parent 560c8f2 commit 6a97be3
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 67 deletions.
121 changes: 80 additions & 41 deletions integration/spadev0/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,19 @@ func AddSpadeTasks(ctx context.Context, requester string, replicasToTest map[int
return nil
}

var spadev0Metadata map[string]string = map[string]string{
"retrieve_type": string(task.Spade),
"max_traverse_depth": "3",
var moduleMetadataMap = map[task.ModuleName]map[string]string{
task.GraphSync: {
"assume_label": "true",
"retrieve_type": "root_block",
},
task.Bitswap: {
"retrieve_type": string(task.Spade),
"max_traverse_depth": "3",
},
task.HTTP: {
"retrieve_type": string(task.Piece),
"retrieve_size": "1048576",
},
}

func prepareTasksForSP(
Expand Down Expand Up @@ -129,7 +139,7 @@ func prepareTasksForSP(
tasks = append(tasks, task.Task{
Requester: requester,
Module: task.Bitswap,
Metadata: spadev0Metadata,
Metadata: moduleMetadataMap[task.Bitswap],
Provider: task.Provider{
ID: spid.String(),
PeerID: providerInfo.PeerId,
Expand All @@ -143,7 +153,29 @@ func prepareTasksForSP(
CID: document.OptionalDagRoot,
},
CreatedAt: time.Now().UTC(),
Timeout: env.GetDuration(env.FilplusIntegrationTaskTimeout, 15*time.Second),
Timeout: env.GetDuration(env.SpadeIntegrationTaskTimeout, 15*time.Second),
})
}

for _, document := range replicas {
tasks = append(tasks, task.Task{
Requester: requester,
Module: task.HTTP,
Metadata: moduleMetadataMap[task.HTTP],
Provider: task.Provider{
ID: document.PieceCID,
PeerID: providerInfo.PeerId,
Multiaddrs: convert.MultiaddrsBytesToStringArraySkippingError(providerInfo.Multiaddrs),
City: location.City,
Region: location.Region,
Country: location.Country,
Continent: location.Continent,
},
Content: task.Content{
CID: document.PieceCID,
},
CreatedAt: time.Now().UTC(),
Timeout: env.GetDuration(env.SpadeIntegrationTaskTimeout, 15*time.Second),
})
}

Expand All @@ -160,43 +192,50 @@ func addErrorResults(
errorCode task.ErrorCode,
errorMessage string,
) []interface{} {
results = append(results, task.Result{
Task: task.Task{
Requester: requester,
Module: "spadev0",
Metadata: spadev0Metadata,
Provider: task.Provider{
ID: spid,
PeerID: providerInfo.PeerId,
Multiaddrs: convert.MultiaddrsBytesToStringArraySkippingError(providerInfo.Multiaddrs),
City: location.City,
Region: location.Region,
Country: location.Country,
Continent: location.Continent,
for module, metadata := range moduleMetadataMap {
newMetadata := make(map[string]string)
for k, v := range metadata {
newMetadata[k] = v
}
newMetadata["client"] = string(module)
results = append(results, task.Result{
Task: task.Task{
Requester: requester,
Module: "spadev0",
Metadata: newMetadata,
Provider: task.Provider{
ID: spid,
PeerID: providerInfo.PeerId,
Multiaddrs: convert.MultiaddrsBytesToStringArraySkippingError(providerInfo.Multiaddrs),
City: location.City,
Region: location.Region,
Country: location.Country,
Continent: location.Continent,
},
CreatedAt: time.Now().UTC(),
Timeout: env.GetDuration(env.SpadeIntegrationTaskTimeout, 15*time.Second)},
Retriever: task.Retriever{
PublicIP: ipInfo.IP,
City: ipInfo.City,
Region: ipInfo.Region,
Country: ipInfo.Country,
Continent: ipInfo.Continent,
ASN: ipInfo.ASN,
ISP: ipInfo.ISP,
Latitude: ipInfo.Latitude,
Longitude: ipInfo.Longitude,
},
Result: task.RetrievalResult{
Success: false,
ErrorCode: errorCode,
ErrorMessage: errorMessage,
TTFB: 0,
Speed: 0,
Duration: 0,
Downloaded: 0,
},
CreatedAt: time.Now().UTC(),
Timeout: env.GetDuration(env.FilplusIntegrationTaskTimeout, 15*time.Second)},
Retriever: task.Retriever{
PublicIP: ipInfo.IP,
City: ipInfo.City,
Region: ipInfo.Region,
Country: ipInfo.Country,
Continent: ipInfo.Continent,
ASN: ipInfo.ASN,
ISP: ipInfo.ISP,
Latitude: ipInfo.Latitude,
Longitude: ipInfo.Longitude,
},
Result: task.RetrievalResult{
Success: false,
ErrorCode: errorCode,
ErrorMessage: errorMessage,
TTFB: 0,
Speed: 0,
Duration: 0,
Downloaded: 0,
},
CreatedAt: time.Now().UTC(),
})
})
}
return results
}
54 changes: 28 additions & 26 deletions pkg/env/env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,47 +2,49 @@ package env

import (
"fmt"
logging "github.com/ipfs/go-log/v2"
"os"
"strconv"
"time"

logging "github.com/ipfs/go-log/v2"
)

type Key string

//nolint:gosec
const (
ProcessModules Key = "PROCESS_MODULES"
ProcessErrorInterval Key = "PROCESS_ERROR_INTERVAL"
TaskWorkerPollInterval Key = "TASK_WORKER_POLL_INTERVAL"
TaskWorkerTimeoutBuffer Key = "TASK_WORKER_TIMEOUT_BUFFER"
LotusAPIUrl Key = "LOTUS_API_URL"
LotusAPIToken Key = "LOTUS_API_TOKEN"
QueueMongoURI Key = "QUEUE_MONGO_URI"
QueueMongoDatabase Key = "QUEUE_MONGO_DATABASE"
ResultMongoURI Key = "RESULT_MONGO_URI"
ResultMongoDatabase Key = "RESULT_MONGO_DATABASE"
FilplusIntegrationBatchSize Key = "FILPLUS_INTEGRATION_BATCH_SIZE"
FilplusIntegrationTaskTimeout Key = "FILPLUS_INTEGRATION_TASK_TIMEOUT"
FilplusIntegrationRandConst Key = "FILPLUS_INTEGRATION_RANDOM_CONSTANT"
StatemarketdealsMongoURI Key = "STATEMARKETDEALS_MONGO_URI"
StatemarketdealsMongoDatabase Key = "STATEMARKETDEALS_MONGO_DATABASE"
StatemarketdealsBatchSize Key = "STATEMARKETDEALS_BATCH_SIZE"
StatemarketdealsInterval Key = "STATEMARKETDEALS_INTERVAL"
PublicIP Key = "_PUBLIC_IP"
AcceptedContinents Key = "ACCEPTED_CONTINENTS"
AcceptedCountries Key = "ACCEPTED_COUNTRIES"
ASN Key = "_ASN"
City Key = "_CITY"
Region Key = "_REGION"
Country Key = "_COUNTRY"
Continent Key = "_CONTINENT"
ASN Key = "_ASN"
Country Key = "_COUNTRY"
FilplusIntegrationBatchSize Key = "FILPLUS_INTEGRATION_BATCH_SIZE"
FilplusIntegrationRandConst Key = "FILPLUS_INTEGRATION_RANDOM_CONSTANT"
FilplusIntegrationTaskTimeout Key = "FILPLUS_INTEGRATION_TASK_TIMEOUT"
IPInfoToken Key = "IPINFO_TOKEN"
ISP Key = "_ISP"
Latitude Key = "_LATITUDE"
LocationCacheTTL Key = "LOCATION_CACHE_TTL"
Longitude Key = "_LONGITUDE"
LotusAPIToken Key = "LOTUS_API_TOKEN"
LotusAPIUrl Key = "LOTUS_API_URL"
ProcessErrorInterval Key = "PROCESS_ERROR_INTERVAL"
ProcessModules Key = "PROCESS_MODULES"
ProviderCacheTTL Key = "PROVIDER_CACHE_TTL"
LocationCacheTTL Key = "LOCATION_CACHE_TTL"
AcceptedContinents Key = "ACCEPTED_CONTINENTS"
AcceptedCountries Key = "ACCEPTED_COUNTRIES"
IPInfoToken Key = "IPINFO_TOKEN"
PublicIP Key = "_PUBLIC_IP"
QueueMongoDatabase Key = "QUEUE_MONGO_DATABASE"
QueueMongoURI Key = "QUEUE_MONGO_URI"
Region Key = "_REGION"
ResultMongoDatabase Key = "RESULT_MONGO_DATABASE"
ResultMongoURI Key = "RESULT_MONGO_URI"
SpadeIntegrationTaskTimeout Key = "SPADE_INTEGRATION_TASK_TIMEOUT"
StatemarketdealsBatchSize Key = "STATEMARKETDEALS_BATCH_SIZE"
StatemarketdealsInterval Key = "STATEMARKETDEALS_INTERVAL"
StatemarketdealsMongoDatabase Key = "STATEMARKETDEALS_MONGO_DATABASE"
StatemarketdealsMongoURI Key = "STATEMARKETDEALS_MONGO_URI"
TaskWorkerPollInterval Key = "TASK_WORKER_POLL_INTERVAL"
TaskWorkerTimeoutBuffer Key = "TASK_WORKER_TIMEOUT_BUFFER"
)

func GetString(key Key, defaultValue string) string {
Expand Down
1 change: 1 addition & 0 deletions pkg/task/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ type RetrieveType string
const (
Any RetrieveType = "any"
Spade RetrieveType = "spade"
Piece RetrieveType = "piece"
)

type Content struct {
Expand Down

0 comments on commit 6a97be3

Please sign in to comment.