package main import ( "encoding/json" "fmt" "io/ioutil" "log" "math" "math/rand" "net/http" "os" "sort" "strings" "sync" "text/template" "time" inventorytools "code.mashffxiv.deadbeef.codes/MashPotato/gilgetter/inventorytools" ) var ( items map[int]*MarketItem // hash map used to quickly guarantee uniqueness of MarketItems itemSlice []MarketItem // rank-ordered slice of the same data in the items map nameUpdateQueue chan int // requests sent to this channel use xivapi https://xivapi.com/ priceUpdateQueue chan int // requests sent to this channel use universalis.app API https://universalis.app/docs/index.html updateRequestWg sync.WaitGroup // Used to wait until the queues are empty indicating all items have up to date names and prices t *template.Template // Used to cache HTML template parsing, done once at startup lastFullPriceUpdateTime time.Time // The last time the prices were updated from Universalis (both start and finish. Start if ongoing, finished when not running) startFullPriceUpdateTime time.Time // The last time the prices were started to be updated lastSortTime time.Time // The last time the itemSlice was sorted by metric lastUniversalisRequestTime time.Time // The last time any request was made to Universalis priceUpdateProgress int // an integer from 0-100 indicating percentage of any ongoing Universalis update. If 100, then the update is complete. world, datacenter, universalisPriceEndpoint string // filter Universalis to this world name priceUpdateErrors map[int]bool // if an item ID exists in here, then there was an error when updating its price. Gets reset after priceUpdateQueue is drained. withdrawItemsAhk []byte // contains the processed ahk template of items that need to be withdrawn for crafting or selling retainers []string // contains name of retainers craftingCheckoutItemIDs []int // The item IDs which were selected on the crafting checkout page, used for shopping list inventories inventorytools.Inventories ) // Object to represent a crafting material / ingredient in a recipe type CraftingMaterial struct { ID int `json:"id"` Quantity int `json:"quantity"` } // Object to represent an item that can be sold on the market type MarketItem struct { ID int `json:"id"` Name string `json:"name,omitempty"` Job string `json:"job,omitempty"` CraftingMetric int `json:"craftingmetric,omitempty"` ListingMetric int `json:"listingmetric,omitempty"` MaterialCost int `json:"materialcost,omitempty"` Velocity int `json:"velocity,omitempty"` CraftingMaterials []CraftingMaterial `json:"craftingmaterials,omitempty"` MarketBoardListing *UniversalisApiMarketBoardListingResponse `json:"marketboardlisting,omitempty"` InStock bool `json:"instock"` // true if item is in character inventory, retainer market listing or retainer bag } // JSON object mapping for HTTP response from XIVAPI type XivApiSearchResponse struct { Pagination struct { Page int `json:"Page"` PageNext interface{} `json:"PageNext"` PagePrev interface{} `json:"PagePrev"` PageTotal int `json:"PageTotal"` Results int `json:"Results"` ResultsPerPage int `json:"ResultsPerPage"` ResultsTotal int `json:"ResultsTotal"` } `json:"Pagination"` Results []struct { ID int `json:"ID"` Icon string `json:"Icon"` Name string `json:"Name"` URL string `json:"Url"` URLType string `json:"UrlType"` Score int `json:"_Score"` } `json:"Results"` SpeedMs int `json:"SpeedMs"` } // JSON object mapping for HTTP response from Universalis type UniversalisApiMarketBoardListingResponse struct { ItemID int `json:"itemID"` WorldID int `json:"worldID"` LastUploadTime int64 `json:"lastUploadTime"` Listings []struct { LastReviewTime int `json:"lastReviewTime"` PricePerUnit int `json:"pricePerUnit"` Quantity int `json:"quantity"` StainID int `json:"stainID"` CreatorName string `json:"creatorName"` CreatorID string `json:"creatorID"` Hq bool `json:"hq"` IsCrafted bool `json:"isCrafted"` ListingID interface{} `json:"listingID"` Materia []interface{} `json:"materia"` OnMannequin bool `json:"onMannequin"` RetainerCity int `json:"retainerCity"` RetainerID string `json:"retainerID"` RetainerName string `json:"retainerName"` SellerID string `json:"sellerID"` Total int `json:"total"` } `json:"listings"` RecentHistory []struct { Hq bool `json:"hq"` PricePerUnit int `json:"pricePerUnit"` Quantity int `json:"quantity"` Timestamp int `json:"timestamp"` BuyerName string `json:"buyerName"` Total int `json:"total"` } `json:"recentHistory"` CurrentAveragePrice float64 `json:"currentAveragePrice"` CurrentAveragePriceNQ float64 `json:"currentAveragePriceNQ"` CurrentAveragePriceHQ float64 `json:"currentAveragePriceHQ"` RegularSaleVelocity float64 `json:"regularSaleVelocity"` NqSaleVelocity float64 `json:"nqSaleVelocity"` HqSaleVelocity float64 `json:"hqSaleVelocity"` AveragePrice float64 `json:"averagePrice"` AveragePriceNQ float64 `json:"averagePriceNQ"` AveragePriceHQ float64 `json:"averagePriceHQ"` MinPrice int `json:"minPrice"` MinPriceNQ int `json:"minPriceNQ"` MinPriceHQ int `json:"minPriceHQ"` MaxPrice int `json:"maxPrice"` MaxPriceNQ int `json:"maxPriceNQ"` MaxPriceHQ int `json:"maxPriceHQ"` StackSizeHistogram struct { Num1 int `json:"1"` } `json:"stackSizeHistogram"` StackSizeHistogramNQ struct { } `json:"stackSizeHistogramNQ"` StackSizeHistogramHQ struct { Num1 int `json:"1"` } `json:"stackSizeHistogramHQ"` WorldName string `json:"worldName"` } // Build queues/channels, load crafting recipes, and launch worker goroutines func init() { log.Println("gilgetter init") // Load application configuration from environment variables // Validate that all required environment variables are set envVars := make(map[string]string) envVars["world"] = os.Getenv("world") envVars["datacenter"] = os.Getenv("datacenter") for key, value := range envVars { if value == "" { log.Fatalf("required environment variable '%s' is not set or has empty value", key) } } retainers = make([]string, 0) for i := 0; true; i++ { retainer := os.Getenv(fmt.Sprintf("retainer_%d", i)) if retainer == "" { break } retainers = append(retainers, retainer) } world = envVars["world"] datacenter = envVars["datacenter"] universalisPriceEndpoint = world updateRequestWg = sync.WaitGroup{} nameUpdateQueue = make(chan int, 8192) priceUpdateQueue = make(chan int, 8192) priceUpdateErrors = make(map[int]bool) items = make(map[int]*MarketItem) loadRecipes() for i := 0; i < 8; i++ { go nameUpdateWorker() go priceUpdateWorker() } go reSortWorker() priceUpdateProgress = 100 // must be set to 100 to allow a price update to occur, to prevent concurrent price update requests err := fullPriceRefresh() if err != nil { log.Fatalf("init failed to refresh price data: %v", err) } lastFullPriceUpdateTime = time.Now() itemSlice = make([]MarketItem, 0) withdrawItemsAhk = make([]byte, 0) } // HTTP routing func main() { // Parse all HTML template files var err error t, err = template.ParseGlob("./templates/*") if err != nil { log.Fatalf("couldn't parse HTML templates: %v", err) } // Serve public static files such as css, js, images http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("public")))) // Download .ahk file http.HandleFunc("/auto-xiv", autoXivHandler) // Page Handlers // Anything that is responsible for the base elements of a viewable web page http.HandleFunc("/", homePageHandler) http.HandleFunc("/stale-items", staleItemsPageHandler) http.HandleFunc("/crafting-checkout", craftingCheckoutPageHandler) http.HandleFunc("/inventory", inventoryPageHandler) http.HandleFunc("/listings", listingsPageHandler) // API Handlers // Anything that is called by the front end but is not a web page http.HandleFunc("/full-price-update", fullPriceUpdateHandler) http.HandleFunc("/price-update", singlePriceUpdateHandler) http.HandleFunc("/status", statusHandler) http.HandleFunc("/items-list", itemsListHandler) http.HandleFunc("/bugged-metric-items-list", buggedMetricItemsListHandler) http.HandleFunc("/stale-items-list", staleItemsListHandler) http.HandleFunc("/search-listing-retainer", searchListingRetainerHandler) http.HandleFunc("/search-listing-craftedby", searchListingCraftedbyHandler) http.HandleFunc("/search-listing-price", searchListingPriceHandler) log.Print("Service listening on :8080") log.Fatal(http.ListenAndServe(":8080", nil)) } // Processes the nameUpdateQueue which contains item IDs that need to be mapped to names // sends request to xivapi func nameUpdateWorker() { for { func() { // anonymous function to allow using defer each loop interation id := <-nameUpdateQueue updateRequestWg.Add(1) defer updateRequestWg.Done() // By this point we know the item name is not found in the cache // and this is the first request to populate it // Sleep for between 1-1.1 seconds to avoid rate limits and play nice with the service providers timeFuzzing := rand.Intn(500) // between 0 and 100ms time.Sleep(time.Millisecond*333 + time.Duration(timeFuzzing)*time.Millisecond) log.Printf("ID: %d - Name Update Request", id) resp, err := http.Get(fmt.Sprintf("https://xivapi.com/search?filters=ID=%d", id)) if err != nil { log.Printf("request to get item ID '%d' name failed: %v", id, err) return } if resp.StatusCode != http.StatusOK { log.Printf("response to get item ID '%d' name expected status is '%d' but got '%d'", id, http.StatusOK, resp.StatusCode) return } body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("failed to read xivapi response body for item ID '%d': %v", id, err) return } xivApiSearchResponse := &XivApiSearchResponse{} err = json.Unmarshal(body, xivApiSearchResponse) if err != nil { log.Printf("failed to unmarshal xivapi response body for item ID '%d': %v", id, err) return } for _, xivApiSearchResult := range xivApiSearchResponse.Results { if xivApiSearchResult.URLType == "Item" { items[id].Name = xivApiSearchResult.Name break } } }() } } // Processes the priceUpdateQueue which contains item IDs that need prices // sends request to universalis func priceUpdateWorker() { for { func() { // anonymous function to allow using defer each loop interation id := <-priceUpdateQueue // Block here until we can read something from the price update queue defer updateRequestWg.Done() // Sleep for avoid rate limits and play nice with the service providers timeFuzzing := rand.Intn(300) // in ms time.Sleep(time.Millisecond*220 + time.Duration(timeFuzzing)*time.Millisecond) // By this point we know the item name is not found in the cache // and this is the first request to populate it resp, err := http.Get(fmt.Sprintf("https://universalis.app/api/%s/%d", universalisPriceEndpoint, id)) if err != nil { log.Printf("request to get item ID '%d' price failed: %v", id, err) return } if resp.StatusCode != http.StatusOK { log.Printf("response to get item ID '%d' price expected status is '%d' but got '%d'", id, http.StatusOK, resp.StatusCode) if _, ok := priceUpdateErrors[id]; !ok { // if this item hasn't already had an error go func() { // wait 5 seconds requeue the item if it failed, in goroutine so the priceUpdateWorker isn't blocked time.Sleep(time.Second * 5) updateRequestWg.Add(1) priceUpdateQueue <- id }() } priceUpdateErrors[id] = true // requeue only works one time return } body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("failed to read universalis response body for item ID '%d': %v", id, err) return } universalisApiMarketBoardListingResponse := &UniversalisApiMarketBoardListingResponse{} err = json.Unmarshal(body, universalisApiMarketBoardListingResponse) if err != nil { log.Printf("failed to unmarshal universalis response body for item ID '%d': %v", id, err) return } items[id].MarketBoardListing = universalisApiMarketBoardListingResponse }() lastUniversalisRequestTime = time.Now() if priceUpdateProgress < 100 { // Only time this is <100 is when a full price update is occurring, we don't want this changed if single items are being updated though priceUpdateProgress = (len(items)-len(priceUpdateQueue))*100/(len(items)) - 1 } } } // Will trigger a resort (destroy and recreate) of the itemSlice when conditions are met func reSortWorker() { for { time.Sleep(time.Second * 30) if time.Now().Before(lastSortTime.Add(time.Second * 60)) { // if the last sort occurred within 60 seconds, ignore continue } if time.Now().Before(lastUniversalisRequestTime.Add(time.Second * 30)) { // if the last universalis price update occurred within 30 seconds, ignore continue // This is because there might be more coming! } if time.Now().After(lastUniversalisRequestTime.Add(time.Second * 60)) { // If the last universalis request was more than a minute ago, we've already sorted continue } sortItemSlice() } } // Loads crafting recipe data from the filesystem // populates the items map func loadRecipes() { // Get all files in the crafting-recipes directory files, err := ioutil.ReadDir("crafting-recipes") if err != nil { log.Fatalf("failed to read crafting-recipes directory: %v", err) } // Loop through all files in the crafting-recipes directory for _, file := range files { if !strings.HasSuffix(file.Name(), ".json") { log.Printf("ignoring non-json file '%s'", file.Name()) continue } log.Printf("loading file - crafting-recipes/%s", file.Name()) jsonItems, err := loadJsonFile(fmt.Sprintf("crafting-recipes/%s", file.Name())) if err != nil { log.Printf("error loading crafting-recipe file '%s': %v", file.Name(), err) continue } // Check for any new unique item encountered gets queued up for // information updates from xivapi and universalis for i, item := range *jsonItems { if _, found := items[item.ID]; !found { // item is new/unique if item.ID == 0 { log.Printf("error found market item with ID of 0 at index %d in file %s", i, file.Name()) continue } items[item.ID] = &MarketItem{ID: item.ID, Job: item.Job, CraftingMaterials: item.CraftingMaterials} items[item.ID].MarketBoardListing = &UniversalisApiMarketBoardListingResponse{} // Queue up any crafting items for information updates as well for j, craftingMaterial := range item.CraftingMaterials { if _, found := items[craftingMaterial.ID]; !found { // item is new/unique if craftingMaterial.ID == 0 { log.Printf("error found crafting item with ID of 0 in market item with ID of '%d', crafting item index %d in file %s", item.ID, j, file.Name()) continue } items[craftingMaterial.ID] = &MarketItem{ID: craftingMaterial.ID} items[craftingMaterial.ID].MarketBoardListing = &UniversalisApiMarketBoardListingResponse{} } } } } } } // Loads a JSON file into a slice of market items func loadJsonFile(file string) (*[]MarketItem, error) { // Load the crafting recipes jsonFile, err := os.Open(file) if err != nil { return nil, fmt.Errorf("failed to open file '%s': %v", file, err) } defer jsonFile.Close() jsonBytes, err := ioutil.ReadAll(jsonFile) if err != nil { return nil, fmt.Errorf("failed to read file '%s' to bytes: %v", file, err) } jsonItems := make([]MarketItem, 0) json.Unmarshal(jsonBytes, &jsonItems) return &jsonItems, nil } // Performs a full price refresh of all items from Universalis func fullPriceRefresh() error { if time.Since(lastFullPriceUpdateTime) < time.Minute*90 { return fmt.Errorf("price refresh cannot occur less than 90 minutes apart") } if priceUpdateProgress != 100 { return fmt.Errorf("price refresh is already in progress") } for _, item := range items { if item.Name == "" { // only update names once unless app restarts nameUpdateQueue <- item.ID } priceUpdateQueue <- item.ID updateRequestWg.Add(1) } startFullPriceUpdateTime = time.Now() lastFullPriceUpdateTime = time.Now() go func() { //anonymous callback function which runs immediately after all items have their prices updated updateRequestWg.Wait() // wait for all items to be updated lastFullPriceUpdateTime = time.Now() sortItemSlice() }() priceUpdateProgress = 0 return nil } // Destroys and recreates the itemSlice using the items map, sorted by weight func sortItemSlice() { log.Printf("metric re-sort triggered") priceUpdateProgress = 100 // We set this to 100 to ensure any Universalis refresh is in the completed state when we sort the items. itemSlice = make([]MarketItem, 0) // Calculate the metric for each item for _, item := range items { if len(item.CraftingMaterials) < 1 { // If there are no crafting materials for an item (i.e. the item is a crafting material itself), we ignore it. continue } //item.Metric = int(item.MarketBoardListing.HqSaleVelocity * float64(item.MarketBoardListing.MinPriceHQ)) // Subtract cost of crafting materials item.MaterialCost = 0 for _, craftingMaterial := range item.CraftingMaterials { if int(items[craftingMaterial.ID].MarketBoardListing.AveragePrice) > 0 { // if we have historical sales data use it item.MaterialCost += (craftingMaterial.Quantity * int(items[craftingMaterial.ID].MarketBoardListing.AveragePrice)) } else { // Otherwise use the current min prices item.MaterialCost += items[craftingMaterial.ID].MarketBoardListing.MinPrice } } // Calculate the base metric based on sale price and velocity if item.MarketBoardListing.MinPriceHQ != 0 { // Use HQ by default item.Velocity = int(math.Round(item.MarketBoardListing.HqSaleVelocity * 10 / 2.857143)) // 2.857143 is max universalis sale velocity value, so this gives us an "out of 10" rating item.CraftingMetric = item.MarketBoardListing.MinPriceHQ item.ListingMetric = item.CraftingMetric*item.Velocity + item.MarketBoardListing.MinPriceHQ } else { // Item is normal quality only for example, housing items item.Velocity = int(math.Round(item.MarketBoardListing.NqSaleVelocity * 10 / 2.857143)) // 2.857143 is max universalis sale velocity value, so this gives us an "out of 10" rating item.CraftingMetric = item.MarketBoardListing.MinPriceNQ item.ListingMetric = item.CraftingMetric*item.Velocity + item.MarketBoardListing.MinPriceNQ } item.CraftingMetric = (item.CraftingMetric-item.MaterialCost)*item.Velocity - item.MaterialCost*2 // Linear scale down outliers outlierWeight := 2.0 if float64(item.MarketBoardListing.MinPriceHQ) > item.MarketBoardListing.AveragePriceHQ { // only scale down when current min price is above average price. Do not scale up ever. item.CraftingMetric = item.CraftingMetric - int(outlierWeight*(float64(item.MarketBoardListing.MinPriceHQ)-item.MarketBoardListing.AveragePriceHQ)) } itemSlice = append(itemSlice, *item) } // Sort the item slice by metric sort.Slice(itemSlice, func(i, j int) bool { return itemSlice[i].CraftingMetric > itemSlice[j].CraftingMetric }) lastSortTime = time.Now() }