MashPotato
/
gilgetter
1
0
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity
gilgetter/main.go

520 lines
20 KiB
Go
Raw Blame History

package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"math"
"math/rand"
"net/http"
"os"
"sort"
"strings"
"sync"
"text/template"
"time"
inventorytools "code.mashffxiv.deadbeef.codes/MashPotato/gilgetter/inventorytools"
)
var (
items map[int]*MarketItem // hash map used to quickly guarantee uniqueness of MarketItems
itemSlice []MarketItem // rank-ordered slice of the same data in the items map
nameUpdateQueue chan int // requests sent to this channel use xivapi https://xivapi.com/
priceUpdateQueue chan int // requests sent to this channel use universalis.app API https://universalis.app/docs/index.html
updateRequestWg sync.WaitGroup // Used to wait until the queues are empty indicating all items have up to date names and prices
t *template.Template // Used to cache HTML template parsing, done once at startup
lastFullPriceUpdateTime time.Time // The last time the prices were updated from Universalis (both start and finish. Start if ongoing, finished when not running)
startFullPriceUpdateTime time.Time // The last time the prices were started to be updated
lastSortTime time.Time // The last time the itemSlice was sorted by metric
lastUniversalisRequestTime time.Time // The last time any request was made to Universalis
priceUpdateProgress int // an integer from 0-100 indicating percentage of any ongoing Universalis update. If 100, then the update is complete.
world, datacenter, universalisPriceEndpoint string // filter Universalis to this world name
priceUpdateErrors map[int]bool // if an item ID exists in here, then there was an error when updating its price. Gets reset after priceUpdateQueue is drained.
withdrawItemsAhk []byte // contains the processed ahk template of items that need to be withdrawn for crafting or selling
retainers []string // contains name of retainers
craftingCheckoutItemIDs []int // The item IDs which were selected on the crafting checkout page, used for shopping list
inventories inventorytools.Inventories
)
// Object to represent a crafting material / ingredient in a recipe
type CraftingMaterial struct {
ID int `json:"id"`
Quantity int `json:"quantity"`
}
// Object to represent an item that can be sold on the market
type MarketItem struct {
ID int `json:"id"`
Name string `json:"name,omitempty"`
Job string `json:"job,omitempty"`
CraftingMetric int `json:"craftingmetric,omitempty"`
ListingMetric int `json:"listingmetric,omitempty"`
MaterialCost int `json:"materialcost,omitempty"`
Velocity int `json:"velocity,omitempty"`
CraftingMaterials []CraftingMaterial `json:"craftingmaterials,omitempty"`
MarketBoardListing *UniversalisApiMarketBoardListingResponse `json:"marketboardlisting,omitempty"`
InStock bool `json:"instock"` // true if item is in character inventory, retainer market listing or retainer bag
}
// JSON object mapping for HTTP response from XIVAPI
type XivApiSearchResponse struct {
Pagination struct {
Page int `json:"Page"`
PageNext interface{} `json:"PageNext"`
PagePrev interface{} `json:"PagePrev"`
PageTotal int `json:"PageTotal"`
Results int `json:"Results"`
ResultsPerPage int `json:"ResultsPerPage"`
ResultsTotal int `json:"ResultsTotal"`
} `json:"Pagination"`
Results []struct {
ID int `json:"ID"`
Icon string `json:"Icon"`
Name string `json:"Name"`
URL string `json:"Url"`
URLType string `json:"UrlType"`
Score int `json:"_Score"`
} `json:"Results"`
SpeedMs int `json:"SpeedMs"`
}
// JSON object mapping for HTTP response from Universalis
type UniversalisApiMarketBoardListingResponse struct {
ItemID int `json:"itemID"`
WorldID int `json:"worldID"`
LastUploadTime int64 `json:"lastUploadTime"`
Listings []struct {
LastReviewTime int `json:"lastReviewTime"`
PricePerUnit int `json:"pricePerUnit"`
Quantity int `json:"quantity"`
StainID int `json:"stainID"`
CreatorName string `json:"creatorName"`
CreatorID string `json:"creatorID"`
Hq bool `json:"hq"`
IsCrafted bool `json:"isCrafted"`
ListingID interface{} `json:"listingID"`
Materia []interface{} `json:"materia"`
OnMannequin bool `json:"onMannequin"`
RetainerCity int `json:"retainerCity"`
RetainerID string `json:"retainerID"`
RetainerName string `json:"retainerName"`
SellerID string `json:"sellerID"`
Total int `json:"total"`
} `json:"listings"`
RecentHistory []struct {
Hq bool `json:"hq"`
PricePerUnit int `json:"pricePerUnit"`
Quantity int `json:"quantity"`
Timestamp int `json:"timestamp"`
BuyerName string `json:"buyerName"`
Total int `json:"total"`
} `json:"recentHistory"`
CurrentAveragePrice float64 `json:"currentAveragePrice"`
CurrentAveragePriceNQ float64 `json:"currentAveragePriceNQ"`
CurrentAveragePriceHQ float64 `json:"currentAveragePriceHQ"`
RegularSaleVelocity float64 `json:"regularSaleVelocity"`
NqSaleVelocity float64 `json:"nqSaleVelocity"`
HqSaleVelocity float64 `json:"hqSaleVelocity"`
AveragePrice float64 `json:"averagePrice"`
AveragePriceNQ float64 `json:"averagePriceNQ"`
AveragePriceHQ float64 `json:"averagePriceHQ"`
MinPrice int `json:"minPrice"`
MinPriceNQ int `json:"minPriceNQ"`
MinPriceHQ int `json:"minPriceHQ"`
MaxPrice int `json:"maxPrice"`
MaxPriceNQ int `json:"maxPriceNQ"`
MaxPriceHQ int `json:"maxPriceHQ"`
StackSizeHistogram struct {
Num1 int `json:"1"`
} `json:"stackSizeHistogram"`
StackSizeHistogramNQ struct {
} `json:"stackSizeHistogramNQ"`
StackSizeHistogramHQ struct {
Num1 int `json:"1"`
} `json:"stackSizeHistogramHQ"`
WorldName string `json:"worldName"`
}
// Build queues/channels, load crafting recipes, and launch worker goroutines
func init() {
log.Println("gilgetter init")
// Load application configuration from environment variables
// Validate that all required environment variables are set
envVars := make(map[string]string)
envVars["world"] = os.Getenv("world")
envVars["datacenter"] = os.Getenv("datacenter")
for key, value := range envVars {
if value == "" {
log.Fatalf("required environment variable '%s' is not set or has empty value", key)
}
}
retainers = make([]string, 0)
for i := 0; true; i++ {
retainer := os.Getenv(fmt.Sprintf("retainer_%d", i))
if retainer == "" {
break
}
retainers = append(retainers, retainer)
}
world = envVars["world"]
datacenter = envVars["datacenter"]
universalisPriceEndpoint = world
updateRequestWg = sync.WaitGroup{}
nameUpdateQueue = make(chan int, 8192)
priceUpdateQueue = make(chan int, 8192)
priceUpdateErrors = make(map[int]bool)
items = make(map[int]*MarketItem)
loadRecipes()
for i := 0; i < 8; i++ {
go nameUpdateWorker()
go priceUpdateWorker()
}
go reSortWorker()
priceUpdateProgress = 100 // must be set to 100 to allow a price update to occur, to prevent concurrent price update requests
err := fullPriceRefresh()
if err != nil {
log.Fatalf("init failed to refresh price data: %v", err)
}
lastFullPriceUpdateTime = time.Now()
itemSlice = make([]MarketItem, 0)
withdrawItemsAhk = make([]byte, 0)
}
// HTTP routing
func main() {
// Parse all HTML template files
var err error
t, err = template.ParseGlob("./templates/*")
if err != nil {
log.Fatalf("couldn't parse HTML templates: %v", err)
}
// Serve public static files such as css, js, images
http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("public"))))
// Download .ahk file
http.HandleFunc("/auto-xiv", autoXivHandler)
// Page Handlers
// Anything that is responsible for the base elements of a viewable web page
http.HandleFunc("/", homePageHandler)
http.HandleFunc("/stale-items", staleItemsPageHandler)
http.HandleFunc("/crafting-checkout", craftingCheckoutPageHandler)
http.HandleFunc("/inventory", inventoryPageHandler)
http.HandleFunc("/listings", listingsPageHandler)
// API Handlers
// Anything that is called by the front end but is not a web page
http.HandleFunc("/full-price-update", fullPriceUpdateHandler)
http.HandleFunc("/price-update", singlePriceUpdateHandler)
http.HandleFunc("/status", statusHandler)
http.HandleFunc("/items-list", itemsListHandler)
http.HandleFunc("/bugged-metric-items-list", buggedMetricItemsListHandler)
http.HandleFunc("/stale-items-list", staleItemsListHandler)
http.HandleFunc("/search-listing-retainer", searchListingRetainerHandler)
http.HandleFunc("/search-listing-craftedby", searchListingCraftedbyHandler)
http.HandleFunc("/search-listing-price", searchListingPriceHandler)
log.Print("Service listening on :8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}
// Processes the nameUpdateQueue which contains item IDs that need to be mapped to names
// sends request to xivapi
func nameUpdateWorker() {
for {
func() { // anonymous function to allow using defer each loop interation
id := <-nameUpdateQueue
updateRequestWg.Add(1)
defer updateRequestWg.Done()
// By this point we know the item name is not found in the cache
// and this is the first request to populate it
// Sleep for between 1-1.1 seconds to avoid rate limits and play nice with the service providers
timeFuzzing := rand.Intn(500) // between 0 and 100ms
time.Sleep(time.Millisecond*333 + time.Duration(timeFuzzing)*time.Millisecond)
log.Printf("ID: %d - Name Update Request", id)
resp, err := http.Get(fmt.Sprintf("https://xivapi.com/search?filters=ID=%d", id))
if err != nil {
log.Printf("request to get item ID '%d' name failed: %v", id, err)
return
}
if resp.StatusCode != http.StatusOK {
log.Printf("response to get item ID '%d' name expected status is '%d' but got '%d'", id, http.StatusOK, resp.StatusCode)
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("failed to read xivapi response body for item ID '%d': %v", id, err)
return
}
xivApiSearchResponse := &XivApiSearchResponse{}
err = json.Unmarshal(body, xivApiSearchResponse)
if err != nil {
log.Printf("failed to unmarshal xivapi response body for item ID '%d': %v", id, err)
return
}
for _, xivApiSearchResult := range xivApiSearchResponse.Results {
if xivApiSearchResult.URLType == "Item" {
items[id].Name = xivApiSearchResult.Name
break
}
}
}()
}
}
// Processes the priceUpdateQueue which contains item IDs that need prices
// sends request to universalis
func priceUpdateWorker() {
for {
func() { // anonymous function to allow using defer each loop interation
id := <-priceUpdateQueue // Block here until we can read something from the price update queue
defer updateRequestWg.Done()
// Sleep for avoid rate limits and play nice with the service providers
timeFuzzing := rand.Intn(300) // in ms
time.Sleep(time.Millisecond*220 + time.Duration(timeFuzzing)*time.Millisecond)
// By this point we know the item name is not found in the cache
// and this is the first request to populate it
resp, err := http.Get(fmt.Sprintf("https://universalis.app/api/%s/%d", universalisPriceEndpoint, id))
if err != nil {
log.Printf("request to get item ID '%d' price failed: %v", id, err)
return
}
if resp.StatusCode != http.StatusOK {
log.Printf("response to get item ID '%d' price expected status is '%d' but got '%d'", id, http.StatusOK, resp.StatusCode)
if _, ok := priceUpdateErrors[id]; !ok { // if this item hasn't already had an error
go func() { // wait 5 seconds requeue the item if it failed, in goroutine so the priceUpdateWorker isn't blocked
time.Sleep(time.Second * 5)
updateRequestWg.Add(1)
priceUpdateQueue <- id
}()
}
priceUpdateErrors[id] = true // requeue only works one time
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("failed to read universalis response body for item ID '%d': %v", id, err)
return
}
universalisApiMarketBoardListingResponse := &UniversalisApiMarketBoardListingResponse{}
err = json.Unmarshal(body, universalisApiMarketBoardListingResponse)
if err != nil {
log.Printf("failed to unmarshal universalis response body for item ID '%d': %v", id, err)
return
}
items[id].MarketBoardListing = universalisApiMarketBoardListingResponse
}()
lastUniversalisRequestTime = time.Now()
if priceUpdateProgress < 100 { // Only time this is <100 is when a full price update is occurring, we don't want this changed if single items are being updated though
priceUpdateProgress = (len(items)-len(priceUpdateQueue))*100/(len(items)) - 1
}
}
}
// Will trigger a resort (destroy and recreate) of the itemSlice when conditions are met
func reSortWorker() {
for {
time.Sleep(time.Second * 30)
if time.Now().Before(lastSortTime.Add(time.Second * 60)) { // if the last sort occurred within 60 seconds, ignore
continue
}
if time.Now().Before(lastUniversalisRequestTime.Add(time.Second * 30)) { // if the last universalis price update occurred within 30 seconds, ignore
continue // This is because there might be more coming!
}
if time.Now().After(lastUniversalisRequestTime.Add(time.Second * 60)) { // If the last universalis request was more than a minute ago, we've already sorted
continue
}
sortItemSlice()
}
}
// Loads crafting recipe data from the filesystem
// populates the items map
func loadRecipes() {
// Get all files in the crafting-recipes directory
files, err := ioutil.ReadDir("crafting-recipes")
if err != nil {
log.Fatalf("failed to read crafting-recipes directory: %v", err)
}
// Loop through all files in the crafting-recipes directory
for _, file := range files {
if !strings.HasSuffix(file.Name(), ".json") {
log.Printf("ignoring non-json file '%s'", file.Name())
continue
}
log.Printf("loading file - crafting-recipes/%s", file.Name())
jsonItems, err := loadJsonFile(fmt.Sprintf("crafting-recipes/%s", file.Name()))
if err != nil {
log.Printf("error loading crafting-recipe file '%s': %v", file.Name(), err)
continue
}
// Check for any new unique item encountered gets queued up for
// information updates from xivapi and universalis
for i, item := range *jsonItems {
if _, found := items[item.ID]; !found { // item is new/unique
if item.ID == 0 {
log.Printf("error found market item with ID of 0 at index %d in file %s", i, file.Name())
continue
}
items[item.ID] = &MarketItem{ID: item.ID, Job: item.Job, CraftingMaterials: item.CraftingMaterials}
items[item.ID].MarketBoardListing = &UniversalisApiMarketBoardListingResponse{}
// Queue up any crafting items for information updates as well
for j, craftingMaterial := range item.CraftingMaterials {
if _, found := items[craftingMaterial.ID]; !found { // item is new/unique
if craftingMaterial.ID == 0 {
log.Printf("error found crafting item with ID of 0 in market item with ID of '%d', crafting item index %d in file %s", item.ID, j, file.Name())
continue
}
items[craftingMaterial.ID] = &MarketItem{ID: craftingMaterial.ID}
items[craftingMaterial.ID].MarketBoardListing = &UniversalisApiMarketBoardListingResponse{}
}
}
}
}
}
}
// Loads a JSON file into a slice of market items
func loadJsonFile(file string) (*[]MarketItem, error) {
// Load the crafting recipes
jsonFile, err := os.Open(file)
if err != nil {
return nil, fmt.Errorf("failed to open file '%s': %v", file, err)
}
defer jsonFile.Close()
jsonBytes, err := ioutil.ReadAll(jsonFile)
if err != nil {
return nil, fmt.Errorf("failed to read file '%s' to bytes: %v", file, err)
}
jsonItems := make([]MarketItem, 0)
json.Unmarshal(jsonBytes, &jsonItems)
return &jsonItems, nil
}
// Performs a full price refresh of all items from Universalis
func fullPriceRefresh() error {
if time.Since(lastFullPriceUpdateTime) < time.Minute*90 {
return fmt.Errorf("price refresh cannot occur less than 90 minutes apart")
}
if priceUpdateProgress != 100 {
return fmt.Errorf("price refresh is already in progress")
}
for _, item := range items {
if item.Name == "" { // only update names once unless app restarts
nameUpdateQueue <- item.ID
}
priceUpdateQueue <- item.ID
updateRequestWg.Add(1)
}
startFullPriceUpdateTime = time.Now()
lastFullPriceUpdateTime = time.Now()
go func() { //anonymous callback function which runs immediately after all items have their prices updated
updateRequestWg.Wait() // wait for all items to be updated
lastFullPriceUpdateTime = time.Now()
sortItemSlice()
}()
priceUpdateProgress = 0
return nil
}
// Destroys and recreates the itemSlice using the items map, sorted by weight
func sortItemSlice() {
log.Printf("metric re-sort triggered")
priceUpdateProgress = 100 // We set this to 100 to ensure any Universalis refresh is in the completed state when we sort the items.
itemSlice = make([]MarketItem, 0)
// Calculate the metric for each item
for _, item := range items {
if len(item.CraftingMaterials) < 1 { // If there are no crafting materials for an item (i.e. the item is a crafting material itself), we ignore it.
continue
}
//item.Metric = int(item.MarketBoardListing.HqSaleVelocity * float64(item.MarketBoardListing.MinPriceHQ))
// Subtract cost of crafting materials
item.MaterialCost = 0
for _, craftingMaterial := range item.CraftingMaterials {
if int(items[craftingMaterial.ID].MarketBoardListing.AveragePrice) > 0 { // if we have historical sales data use it
item.MaterialCost += (craftingMaterial.Quantity * int(items[craftingMaterial.ID].MarketBoardListing.AveragePrice))
} else { // Otherwise use the current min prices
item.MaterialCost += items[craftingMaterial.ID].MarketBoardListing.MinPrice
}
}
// Calculate the base metric based on sale price and velocity
if item.MarketBoardListing.MinPriceHQ != 0 {
// Use HQ by default
item.Velocity = int(math.Round(item.MarketBoardListing.HqSaleVelocity * 10 / 2.857143)) // 2.857143 is max universalis sale velocity value, so this gives us an "out of 10" rating
item.CraftingMetric = item.MarketBoardListing.MinPriceHQ
item.ListingMetric = item.CraftingMetric*item.Velocity + item.MarketBoardListing.MinPriceHQ
} else {
// Item is normal quality only for example, housing items
item.Velocity = int(math.Round(item.MarketBoardListing.NqSaleVelocity * 10 / 2.857143)) // 2.857143 is max universalis sale velocity value, so this gives us an "out of 10" rating
item.CraftingMetric = item.MarketBoardListing.MinPriceNQ
item.ListingMetric = item.CraftingMetric*item.Velocity + item.MarketBoardListing.MinPriceNQ
}
item.CraftingMetric = (item.CraftingMetric-item.MaterialCost)*item.Velocity - item.MaterialCost*2
// Linear scale down outliers
outlierWeight := 2.0
if float64(item.MarketBoardListing.MinPriceHQ) > item.MarketBoardListing.AveragePriceHQ { // only scale down when current min price is above average price. Do not scale up ever.
item.CraftingMetric = item.CraftingMetric - int(outlierWeight*(float64(item.MarketBoardListing.MinPriceHQ)-item.MarketBoardListing.AveragePriceHQ))
}
itemSlice = append(itemSlice, *item)
}
// Sort the item slice by metric
sort.Slice(itemSlice, func(i, j int) bool {
return itemSlice[i].CraftingMetric > itemSlice[j].CraftingMetric
})
lastSortTime = time.Now()
}
Reference in New Issue View Git Blame Copy Permalink