diff --git a/docs/go.md b/docs/go.md index 6997bfa..ce09a61 100644 --- a/docs/go.md +++ b/docs/go.md @@ -57,6 +57,12 @@ You can fetch various types of data using the following methods: allAirports, err := client.GetAirports(ctx, nil) ``` + FR24 lists airports one page per country (and per state for the largest + countries), so this makes one request per page: a handful for a few + countries, a few hundred for every airport, `MaxWorkers` at a time. The + listing carries the name, codes and country only; use `GetAirport(ctx, code)` + for the position and altitude of an airport. + - **Airlines list:** ```go diff --git a/docs/nodejs.md b/docs/nodejs.md index 8fc3264..b1cfca1 100644 --- a/docs/nodejs.md +++ b/docs/nodejs.md @@ -36,6 +36,12 @@ You can fetch various types of data using the following methods: let airports = await frApi.getAirports(...); // Returns a list of Airport objects ``` + FR24 lists airports one page per country (and per state for the largest + countries), so this makes one request per page: a handful for a few + countries, a few hundred for every airport, `maxWorkers` at a time. The + listing carries the name, codes and country only; use `getAirport(code)` + for the position and altitude of an airport. + - **Airlines list:** ```javascript diff --git a/docs/python.md b/docs/python.md index e7a3fc7..d9ec6a8 100644 --- a/docs/python.md +++ b/docs/python.md @@ -36,6 +36,12 @@ You can fetch various types of data using the following methods: airports = fr_api.get_airports(...) # Returns a list of Airport objects ``` + FR24 lists airports one page per country (and per state for the largest + countries), so this makes one request per page: a handful for a few + countries, a few hundred for every airport, `max_workers` at a time. The + listing carries the name, codes and country only; use `get_airport(code)` + for the position and altitude of an airport. + - **Airlines list:** ```python diff --git a/go/flightradarapi/api.go b/go/flightradarapi/api.go index 19e1d3d..0215037 100644 --- a/go/flightradarapi/api.go +++ b/go/flightradarapi/api.go @@ -9,6 +9,7 @@ import ( "math" "net/http" "net/url" + "slices" "strings" "sync" "time" @@ -286,23 +287,234 @@ func (c *Client) GetAirportDisruptions(ctx context.Context) (map[string]any, err // GetAirports returns every airport, or only those of the given countries. Pass // nil for every airport; an empty non-nil slice selects none. +// +// FR24 lists airports one page per country, and one per state for the largest +// countries, so this fetches one page per country asked for (a few hundred for +// every airport), MaxWorkers at a time. The listing carries the name, codes and +// country only: use [Client.GetAirport] for the position and altitude of an +// airport. func (c *Client) GetAirports(ctx context.Context, countries []Country) ([]*Airport, error) { if countries != nil && len(countries) == 0 { return []*Airport{}, nil } - response, err := c.client.request(ctx, c.endpoints.airportsJSON, requestOptions{ - headers: jsonHeaders, + // The index names every country, and declares the asset version the JSON + // requests below have to quote. + response, err := c.client.request(ctx, c.endpoints.airportsData, requestOptions{ + headers: htmlHeaders, + timeout: c.Timeout, + }) + + if err != nil { + return nil, err + } + + indexPage, _ := parseInertiaPage(response.Body) + version := toText(indexPage["version"]) + + var slugs []string + + if countries == nil { + for _, country := range parseAirportsIndex(indexPage) { + slugs = append(slugs, country.Slug) + } + } else { + for _, country := range countries { + slugs = append(slugs, countryToSlug(string(country))) + } + } + + slugs = uniqueInOrder(slices.DeleteFunc(slugs, func(slug string) bool { return slug == "" })) + + countryURLs := make([]string, len(slugs)) + + for index, slug := range slugs { + countryURLs[index] = c.endpoints.airportsByCountryURL(slug) + } + + countryPages, err := c.getAirportPages(ctx, countryURLs, version) + + if err != nil { + return nil, err + } + + // Second pass for the countries split by state, every state at once. A + // state path is relative to the site, so it is resolved against the page + // that listed it rather than against a fixed host. + var stateURLs []string + var stateOwners []int + + for owner, page := range countryPages { + for _, path := range page.statePaths { + stateURLs = append(stateURLs, resolveReference(countryURLs[owner], path)) + stateOwners = append(stateOwners, owner) + } + } + + statePages, err := c.getAirportPages(ctx, stateURLs, version) + + if err != nil { + return nil, err + } + + // Grouped by country, in the order asked for, each country's states after + // it. + airportsByCountry := make([][]*Airport, len(countryPages)) + + for index, page := range countryPages { + airportsByCountry[index] = page.airports + } + for index, page := range statePages { + owner := stateOwners[index] + airportsByCountry[owner] = append(airportsByCountry[owner], page.airports...) + } + + airports := []*Airport{} + + for _, found := range airportsByCountry { + airports = append(airports, found...) + } + + // A country FR24 does not know answers with the index page, which lists + // neither airports nor states. + var missing []string + + for index, page := range countryPages { + if len(page.airports) == 0 && len(page.statePaths) == 0 { + missing = append(missing, slugs[index]) + } + } + if len(missing) > 0 { + log().Warn("GetAirports: no airports found for some countries — check the Country constants", + "countries", strings.Join(missing, ", ")) + } + return airports, nil +} + +// airportsPage is what one airports listing page said: its airports, or the +// paths of the state pages it is split into. +type airportsPage struct { + airports []*Airport + statePaths []string +} + +// getAirportPages fetches and parses the airport listing pages at urls, in +// order, MaxWorkers at a time. The first failure fails the whole call. +func (c *Client) getAirportPages(ctx context.Context, urls []string, version string) ([]airportsPage, error) { + pages := make([]airportsPage, len(urls)) + + if len(urls) == 0 { + return pages, nil + } + + // Clamped rather than skipped: MaxWorkers is public, and a zero there must + // not deadlock the crawl. + workers := make(chan struct{}, max(c.MaxWorkers, 1)) + var group sync.WaitGroup + var once sync.Once + var firstErr error + + for index, target := range urls { + workers <- struct{}{} + group.Add(1) + + go func() { + defer group.Done() + defer func() { <-workers }() + + payload, err := c.getInertiaPage(ctx, target, version) + + if err != nil { + once.Do(func() { firstErr = err }) + return + } + pages[index].airports, pages[index].statePaths = parseAirportsPage(payload) + }() + } + group.Wait() + + if firstErr != nil { + return nil, firstErr + } + return pages, nil +} + +// getInertiaPage fetches a FR24 data page as its Inertia page object, falling +// back to the HTML. It returns the decoded JSON body, or the HTML bytes, for +// parseInertiaPage to read. +// +// With the asset version FR24 answers the bare page JSON, a fraction of the +// HTML. A 409 means the assets were redeployed since the version was read: the +// HTML carries the same page object and needs no version. +func (c *Client) getInertiaPage(ctx context.Context, target, version string) (any, error) { + if version != "" { + response, err := c.client.requestStandalone(ctx, target, requestOptions{ + headers: withHeaders(inertiaHeaders, map[string]string{"x-inertia-version": version}), + allowedErrorCodes: []int{409}, + timeout: c.Timeout, + }) + + if err != nil { + return nil, err + } + + if response.StatusCode != 409 { + if !response.IsJSON() { + return response.Body, nil + } + + // Decoded as any value, not as an object: a body that is not a + // page object is the parser's warning, not a failed call. + var page any + + if err := json.Unmarshal([]byte(response.Text()), &page); err != nil { + return nil, fmt.Errorf("%w: could not parse the JSON body of %s: %w", ErrFlightRadar, response.URL, err) + } + return page, nil + } + } + + response, err := c.client.requestStandalone(ctx, target, requestOptions{ + headers: htmlHeaders, timeout: c.Timeout, }) if err != nil { return nil, err } + return response.Body, nil +} - // The raw body, not JSON(): an HTML body reaches the parser's guard, and a - // malformed JSON body still returns empty rather than failing the call. - return parseAirportsJSON(response.Body, countries), nil +// uniqueInOrder drops repeated values, keeping the first occurrence of each. +func uniqueInOrder(values []string) []string { + seen := make(map[string]bool, len(values)) + unique := values[:0:0] + + for _, value := range values { + if !seen[value] { + seen[value] = true + unique = append(unique, value) + } + } + return unique +} + +// resolveReference resolves a path FR24 printed on a page against that page's +// URL, so "/data/airports/brazil/sp" lands on the host the page came from. A +// reference that will not parse is returned as given. +func resolveReference(base, reference string) string { + baseURL, err := url.Parse(base) + + if err != nil { + return reference + } + + referenceURL, err := url.Parse(reference) + + if err != nil { + return reference + } + return baseURL.ResolveReference(referenceURL).String() } // GetBookmarks returns the bookmarks of the logged-in account. diff --git a/go/flightradarapi/api_test.go b/go/flightradarapi/api_test.go index b5449d8..31eca65 100644 --- a/go/flightradarapi/api_test.go +++ b/go/flightradarapi/api_test.go @@ -1,16 +1,22 @@ package flightradarapi import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "math" "net/http" "net/http/cookiejar" "net/http/httptest" "net/url" + "os" + "path/filepath" + "slices" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -48,7 +54,8 @@ func testEndpoints(base string) endpoints { historicalData: base + "/download/?flight=%s&file=%s&trailLimit=0&history=%d", apiAirportData: base + "/common/v1/airport.json", airportData: base + "/airports/traffic-stats/?airport=%s", - airportsJSON: base + "/_json/airports.php", + airportsData: base + "/data/airports", + airportsByCountry: base + "/data/airports/%s", airportDisruption: base + "/webapi/v1/airport-disruptions", airlinesData: base + "/data/airlines", volcanicEruptionData: base + "/weather/volcanic", @@ -97,54 +104,399 @@ func TestGetAirlinesParsesTheListingPage(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(airlines) != 5 { - t.Errorf("got %d airlines, want 5", len(airlines)) + if len(airlines) != 6 { + t.Errorf("got %d airlines, want 6", len(airlines)) } } -func TestGetAirportsFiltersByCountry(t *testing.T) { +// --- GetAirports --- +// +// FR24 lists airports one Inertia page per country, and per state for the +// largest countries. These tests pin down which pages the crawl asks for and +// how it reads them, against the bundled fixtures, mirroring +// python/tests/test_airports_offline.py. + +// Every page the fixtures model, by the path FR24 serves it at. +var airportPages = map[string]string{ + "/data/airports": "airports_index.html", + "/data/airports/uruguay": "airports_country.json", + "/data/airports/brazil": "airports_country_states.json", + "/data/airports/brazil/sp": "airports_state_sp.json", + "/data/airports/brazil/rj": "airports_state_rj.json", + "/data/airports/myanmar-burma": "airports_country.html", +} + +const fixtureAssetVersion = "919b94d2a33f365a34f99828f6eeec6a" + +// airportsCall is one request the fake FR24 saw. +type airportsCall struct { + path string + headers http.Header +} + +// airportsServer serves the fixture pages the way FR24 does. A request carrying +// the inertia headers gets the page JSON when the version matches and a 409 +// otherwise; a plain request gets the HTML. The index page only exists as HTML, +// like the real one as the crawl fetches it. +type airportsServer struct { + t *testing.T + version string + + mu sync.Mutex + calls []airportsCall +} + +func newAirportsServer(t *testing.T, version string) *airportsServer { + t.Helper() + return &airportsServer{t: t, version: version} +} + +func (s *airportsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.calls = append(s.calls, airportsCall{path: r.URL.Path, headers: r.Header.Clone()}) + s.mu.Unlock() + + fixture, ok := airportPages[r.URL.Path] + + if !ok { + fixture = "airports_index.html" + } + + if r.Header.Get("X-Inertia") == "true" { + if r.Header.Get("X-Inertia-Version") != s.version { + w.WriteHeader(http.StatusConflict) + return + } + if strings.HasSuffix(fixture, ".json") { + w.Header().Set("Content-Type", "application/json") + w.Write(loadFixture(s.t, fixture)) + return + } + + page, ok := parseInertiaPage(loadFixture(s.t, fixture)) + + if !ok { + s.t.Errorf("fixture %s carries no page object", fixture) + } + writeJSON(s.t, w, page) + return + } + + if strings.HasSuffix(fixture, ".json") { + fixture = strings.TrimSuffix(fixture, ".json") + ".html" + + if _, err := os.Stat(filepath.Join("testdata", fixture)); err != nil { + fixture = "airports_country.html" + } + } + w.Header().Set("Content-Type", "text/html") + w.Write(loadFixture(s.t, fixture)) +} + +// paths are the paths requested so far, in order. +func (s *airportsServer) paths() []string { + s.mu.Lock() + defer s.mu.Unlock() + paths := make([]string, 0, len(s.calls)) + + for _, call := range s.calls { + paths = append(paths, call.path) + } + return paths +} + +func (s *airportsServer) call(index int) airportsCall { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls[index] +} + +// newSerialAirportsClient wires a client to the fake, one request at a time, so +// the order of the requests is the order of the crawl rather than the order the +// goroutines happened to run in. +func newSerialAirportsClient(t *testing.T, server *airportsServer) *Client { + t.Helper() + client := newTestClient(t, server) + client.MaxWorkers = 1 + return client +} + +func TestGetAirportsCrawlsTheCountryThenItsStates(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + + airports, err := newSerialAirportsClient(t, server).GetAirports(context.Background(), []Country{CountryBrazil}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := iatas(airports); !slices.Equal(got, []string{"GIG", "GRU", "CGH"}) { + t.Errorf("got %v, want GIG, GRU, CGH", got) + } + for _, airport := range airports { + if airport.Country != "Brazil" { + t.Errorf("%s: got country %q, want Brazil", airport.IATA, airport.Country) + } + } + + want := []string{"/data/airports", "/data/airports/brazil", "/data/airports/brazil/rj", "/data/airports/brazil/sp"} + + if got := server.paths(); !slices.Equal(got, want) { + t.Errorf("got requests %v, want %v", got, want) + } +} + +func TestGetAirportsAsksForThePageJSONWithTheIndexVersion(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + + if _, err := newTestClient(t, server).GetAirports(context.Background(), []Country{CountryUruguay}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := server.paths(); !slices.Equal(got, []string{"/data/airports", "/data/airports/uruguay"}) { + t.Fatalf("got requests %v, want the index then the country", got) + } + + index, country := server.call(0), server.call(1) + + if index.headers.Get("X-Inertia") != "" || !strings.Contains(index.headers.Get("Accept"), "text/html") { + t.Errorf("index: got headers %v, want the html headers", index.headers) + } + if country.headers.Get("X-Inertia") != "true" || country.headers.Get("X-Inertia-Version") != fixtureAssetVersion { + t.Errorf("country: got headers %v, want the inertia headers with the index version", country.headers) + } + if country.headers.Get("X-Requested-With") != "XMLHttpRequest" || country.headers.Get("Accept") != "application/json" { + t.Errorf("country: got headers %v, want an XMLHttpRequest for JSON", country.headers) + } +} + +func TestGetAirportsFallsBackToHTMLWhenTheAssetsWereRedeployed(t *testing.T) { + server := newAirportsServer(t, "something-newer") + + airports, err := newTestClient(t, server).GetAirports(context.Background(), []Country{"uruguay"}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // The index answered with a stale version, the JSON request got a 409 and + // the HTML page was read instead. + if len(airports) != 1 { + t.Errorf("got %d airports, want the one on the HTML page", len(airports)) + } + + want := []string{"/data/airports", "/data/airports/uruguay", "/data/airports/uruguay"} + + if got := server.paths(); !slices.Equal(got, want) { + t.Fatalf("got requests %v, want %v", got, want) + } + + retry := server.call(2).headers + + if retry.Get("X-Inertia") != "" || !strings.Contains(retry.Get("Accept"), "text/html") { + t.Errorf("got headers %v, want the html headers", retry) + } +} + +func TestGetAirportsReadsTheHTMLWhenTheIndexDeclaresNoVersion(t *testing.T) { + // An index with no page object leaves the crawl without a version, so it + // asks for the HTML straight away rather than fishing for a 409. mux := http.NewServeMux() - mux.HandleFunc("/_json/airports.php", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.Write(loadFixture(t, "airports.json")) + server := newAirportsServer(t, fixtureAssetVersion) + mux.HandleFunc("/data/airports", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte("maintenance")) }) + mux.Handle("/data/airports/", server) - client := newTestClient(t, mux) - airports, err := client.GetAirports(context.Background(), []Country{CountryBrazil}) + airports, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryMyanmarBurma}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := iatas(airports); !slices.Equal(got, []string{"RGN"}) { + t.Errorf("got %v, want RGN", got) + } + if got := server.paths(); !slices.Equal(got, []string{"/data/airports/myanmar-burma"}) { + t.Fatalf("got requests %v, want the country page once", got) + } + if headers := server.call(0).headers; headers.Get("X-Inertia") != "" { + t.Errorf("got headers %v, want the html headers", headers) + } +} + +func TestGetAirportsWithoutCountriesWalksTheWholeIndex(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + + airports, err := newSerialAirportsClient(t, server).GetAirports(context.Background(), nil) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(airports) != 4 { - t.Errorf("got %d Brazilian airports, want 4", len(airports)) + if got := iatas(airports); !slices.Equal(got, []string{"GIG", "GRU", "CGH", "RGN", "CYR", "MVD"}) { + t.Errorf("got %v, want every airport grouped by country", got) + } + + want := []string{ + "/data/airports", + "/data/airports/brazil", "/data/airports/myanmar-burma", "/data/airports/uruguay", + "/data/airports/brazil/rj", "/data/airports/brazil/sp", + } + + if got := server.paths(); !slices.Equal(got, want) { + t.Errorf("got requests %v, want %v", got, want) + } +} + +func TestGetAirportsGroupsByCountryWhateverOrderThePagesArriveIn(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + client := newTestClient(t, server) + client.MaxWorkers = 16 + + // Repeated so a scheduling order that happens to match does not pass it. + for range 5 { + airports, err := client.GetAirports(context.Background(), []Country{CountryUruguay, CountryBrazil, CountryMyanmarBurma}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := iatas(airports); !slices.Equal(got, []string{"CYR", "MVD", "GIG", "GRU", "CGH", "RGN"}) { + t.Fatalf("got %v, want the countries in the order asked for", got) + } } +} + +func TestGetAirportsAcceptsMixedSpellingsAndDropsRepeats(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) - every, err := client.GetAirports(context.Background(), nil) + airports, err := newTestClient(t, server).GetAirports(context.Background(), + []Country{"Uruguay", CountryUruguay, "uruguay", ""}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(every) != 8 { - t.Errorf("got %d airports for nil, want all 8", len(every)) + if got := iatas(airports); !slices.Equal(got, []string{"CYR", "MVD"}) { + t.Errorf("got %v, want CYR and MVD once", got) + } + if got := server.paths(); !slices.Equal(got, []string{"/data/airports", "/data/airports/uruguay"}) { + t.Errorf("got requests %v, want the country page once", got) } } func TestGetAirportsWithAnEmptyFilterMakesNoRequest(t *testing.T) { - var calls atomic.Int32 + server := newAirportsServer(t, fixtureAssetVersion) + + airports, err := newTestClient(t, server).GetAirports(context.Background(), []Country{}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if airports == nil || len(airports) != 0 || len(server.paths()) != 0 { + t.Errorf("got %v after %v, want an empty slice and no request", airports, server.paths()) + } +} + +func TestGetAirportsUnknownCountryReturnsNothingAndWarns(t *testing.T) { + var logged bytes.Buffer + SetLogger(slog.New(slog.NewTextHandler(&logged, nil))) + t.Cleanup(func() { SetLogger(nil) }) + + server := newAirportsServer(t, fixtureAssetVersion) + airports, err := newTestClient(t, server).GetAirports(context.Background(), []Country{"atlantis"}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if airports == nil || len(airports) != 0 { + t.Errorf("got %v, want an empty slice", airports) + } + + output := logged.String() + + if !strings.Contains(output, "level=WARN") || !strings.Contains(output, "no airports found") || + !strings.Contains(output, "atlantis") { + t.Errorf("got %q, want a warning naming atlantis", output) + } +} + +func TestGetAirportsFailsWhenAPageCannotBeFetched(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/_json/airports.php", func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) + server := newAirportsServer(t, fixtureAssetVersion) + mux.Handle("/", server) + mux.HandleFunc("/data/airports/brazil/sp", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + + _, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryBrazil}) + var statusErr *StatusError + + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusInternalServerError { + t.Fatalf("got %v, want the 500 reported", err) + } +} + +func TestGetAirportsFailsWhenTheIndexCannotBeFetched(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/data/airports", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }) + + if _, err := newTestClient(t, mux).GetAirports(context.Background(), nil); err == nil { + t.Fatal("got no error, want the 503 reported") + } +} + +func TestGetAirportsRejectsAMalformedPageJSON(t *testing.T) { + mux := http.NewServeMux() + server := newAirportsServer(t, fixtureAssetVersion) + mux.Handle("/", server) + mux.HandleFunc("/data/airports/uruguay", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - w.Write(loadFixture(t, "airports.json")) + w.Write([]byte(`{"component": "Data/AirportsByCountry", "props": `)) }) - airports, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{}) + _, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryUruguay}) + + if !errors.Is(err, ErrFlightRadar) || !strings.Contains(err.Error(), "could not parse the JSON body") { + t.Fatalf("got %v, want the malformed body reported", err) + } +} + +func TestGetAirportsTreatsAPageJSONThatIsNotAnObjectAsEmpty(t *testing.T) { + mux := http.NewServeMux() + server := newAirportsServer(t, fixtureAssetVersion) + mux.Handle("/", server) + mux.HandleFunc("/data/airports/uruguay", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`null`)) + }) + + airports, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryUruguay}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(airports) != 0 || calls.Load() != 0 { - t.Errorf("got %d airports after %d calls, want neither", len(airports), calls.Load()) + if len(airports) != 0 { + t.Errorf("got %d airports, want none", len(airports)) + } +} + +func TestGetAirportsResolvesStatePathsAgainstTheCountryPage(t *testing.T) { + // FR24 prints state paths without a host; an absolute URL must survive too. + base := "https://www.flightradar24.com/data/airports/brazil" + + cases := map[string]string{ + "/data/airports/brazil/sp": "https://www.flightradar24.com/data/airports/brazil/sp", + "https://www.flightradar24.com/data/airports/x/y": "https://www.flightradar24.com/data/airports/x/y", + "rj": "https://www.flightradar24.com/data/airports/rj", + "::not a url": "::not a url", + } + + for reference, want := range cases { + if got := resolveReference(base, reference); got != want { + t.Errorf("%q: got %q, want %q", reference, got, want) + } + } + if got := resolveReference("::broken base", "/data/airports/brazil/sp"); got != "/data/airports/brazil/sp" { + t.Errorf("got %q, want the path as given", got) } } diff --git a/go/flightradarapi/core.go b/go/flightradarapi/core.go index 08ba314..209f338 100644 --- a/go/flightradarapi/core.go +++ b/go/flightradarapi/core.go @@ -30,7 +30,8 @@ type endpoints struct { apiAirportData string airportData string - airportsJSON string + airportsData string + airportsByCountry string airportDisruption string airlinesData string @@ -56,9 +57,15 @@ func defaultEndpoints() endpoints { historicalData: flightRadarBaseURL + "/download/?flight=%s&file=%s&trailLimit=0&history=%d", - apiAirportData: apiFlightRadarBaseURL + "/airport.json", - airportData: flightRadarBaseURL + "/airports/traffic-stats/?airport=%s", - airportsJSON: flightRadarBaseURL + "/_json/airports.php", + apiAirportData: apiFlightRadarBaseURL + "/airport.json", + airportData: flightRadarBaseURL + "/airports/traffic-stats/?airport=%s", + // The listing is split by country, and by state for the largest + // countries. These are Inertia pages: the data rides JSON-encoded in the + // data-page attribute of the HTML, or comes as the whole body when the + // request carries inertiaHeaders and the asset version the index page + // declares. + airportsData: flightRadarBaseURL + "/data/airports", + airportsByCountry: flightRadarBaseURL + "/data/airports/%s", airportDisruption: flightRadarBaseURL + "/webapi/v1/airport-disruptions", airlinesData: flightRadarBaseURL + "/data/airlines", @@ -97,6 +104,10 @@ func (e endpoints) countryFlagURL(slug string) string { return fmt.Sprintf(e.countryFlag, url.PathEscape(slug)) } +func (e endpoints) airportsByCountryURL(slug string) string { + return fmt.Sprintf(e.airportsByCountry, url.PathEscape(slug)) +} + func (e endpoints) airlineLogoURL(iata, icao string) string { return fmt.Sprintf(e.airlineLogo, url.PathEscape(iata), url.PathEscape(icao)) } @@ -130,6 +141,13 @@ var ( imageHeaders = withHeaders(baseHeaders, map[string]string{ "accept": "image/gif, image/jpg, image/jpeg, image/png", }) + // inertiaHeaders ask an Inertia page for its page object instead of the + // HTML around it. The request must also carry an x-inertia-version matching + // the deployed assets, or FR24 answers 409. + inertiaHeaders = withHeaders(jsonHeaders, map[string]string{ + "x-inertia": "true", + "x-requested-with": "XMLHttpRequest", + }) htmlHeaders = map[string]string{ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8," + diff --git a/go/flightradarapi/parsers.go b/go/flightradarapi/parsers.go index 57fd6b8..83f2f9c 100644 --- a/go/flightradarapi/parsers.go +++ b/go/flightradarapi/parsers.go @@ -4,11 +4,9 @@ import ( "bytes" "encoding/json" "errors" - "io" "math" "reflect" "regexp" - "slices" "strconv" "strings" "unicode" @@ -25,6 +23,16 @@ var numericPattern = regexp.MustCompile(`^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+ // U+001C-U+001F and U+FEFF. const surroundingSpace = " \t\n\r\f\v" +// airlineCodePattern is an airline code badge: a two-letter IATA code or a +// three-letter ICAO code. ASCII only, so a stray Unicode letter does not pass +// for a code. +var airlineCodePattern = regexp.MustCompile(`^[A-Z0-9]{2,3}$`) + +// aircraftCountPattern is a fleet size as the page prints it: "340" or "3,878". +var aircraftCountPattern = regexp.MustCompile(`^[0-9]{1,3}(?:[,.][0-9]{3})*$|^[0-9]+$`) + +const airlinesPathPrefix = "/data/airlines/" + // Airline is one row of the airlines listing. The JSON tags match the keys the // Python and Node.js ports use, so NumAircrafts serialises as "n_aircrafts". type Airline struct { @@ -35,6 +43,11 @@ type Airline struct { } // parseAirlinesHTML parses the airlines listing page into airline records. +// +// Every airline is a link to its data page, with the IATA and ICAO codes as +// badges next to the name. The "Largest fleets" section repeats a few airlines +// with their aircraft count; an airline the page lists twice becomes one entry +// carrying everything the page said about it. func parseAirlinesHTML(page []byte) []Airline { document, err := html.Parse(bytes.NewReader(page)) @@ -43,71 +56,114 @@ func parseAirlinesHTML(page []byte) []Airline { return []Airline{} } - tbody := findElement(document, func(node *html.Node) bool { return node.Data == "tbody" }) + links := findElements(document, func(node *html.Node) bool { + return node.Data == "a" && strings.HasPrefix(attr(node, "href"), airlinesPathPrefix) + }) - if tbody == nil { - log().Warn("parseAirlinesHTML: no in response — FR24 page layout may have changed") + if len(links) == 0 { + log().Warn("parseAirlinesHTML: no airline links in response — FR24 page layout may have changed") return []Airline{} } airlines := []Airline{} + indexByHref := map[string]int{} - for _, row := range findElements(tbody, func(node *html.Node) bool { return node.Data == "tr" }) { - cells := findElements(row, func(node *html.Node) bool { return node.Data == "td" }) - notranslate := findElement(row, func(node *html.Node) bool { - return node.Data == "td" && hasClass(node, "notranslate") - }) + for _, link := range links { + name := airlineName(link) - if notranslate == nil { + if len(name) < 2 { continue } - link := findElement(notranslate, func(node *html.Node) bool { - return node.Data == "a" && strings.HasPrefix(attr(node, "href"), "/data/airlines") - }) + var iata, icao string - if link == nil { - continue + for _, text := range leafTexts(link, "span") { + switch { + case !airlineCodePattern.MatchString(text): + case len(text) == 2 && iata == "": + iata = text + case len(text) == 3 && icao == "": + icao = text + } } - name := textContent(link) - - if len(name) < 2 { + // Section links ("All") point under /data/airlines/ too; a code tells + // an airline apart from them. + if iata == "" && icao == "" { continue } - airline := Airline{Name: name} + var numAircrafts *int - if len(cells) >= 4 { - codes := textContent(cells[3]) + for _, text := range leafTexts(link, "strong") { + if aircraftCountPattern.MatchString(text) { + digits := strings.Map(func(char rune) rune { + if char >= '0' && char <= '9' { + return char + } + return -1 + }, text) - switch { - case strings.Contains(codes, " / "): - if parts := strings.Split(codes, " / "); len(parts) == 2 { - airline.IATA = strings.TrimSpace(parts[0]) - airline.ICAO = strings.TrimSpace(parts[1]) + if count, err := strconv.Atoi(digits); err == nil { + numAircrafts = &count + break } - case len(codes) == 2: - airline.IATA = codes - case len(codes) == 3: - airline.ICAO = codes } } - if len(cells) >= 5 { - if text := textContent(cells[4]); text != "" { - field := strings.TrimSpace(strings.SplitN(text, " ", 2)[0]) + href := attr(link, "href") + index, seen := indexByHref[href] - if count, err := strconv.Atoi(field); err == nil { - airline.NumAircrafts = &count - } - } + if !seen { + index = len(airlines) + indexByHref[href] = index + airlines = append(airlines, Airline{Name: name}) + } + + airline := &airlines[index] + + if airline.ICAO == "" { + airline.ICAO = icao + } + if airline.IATA == "" { + airline.IATA = iata + } + if airline.NumAircrafts == nil { + airline.NumAircrafts = numAircrafts } - airlines = append(airlines, airline) } return airlines } +// airlineName is the airline name of a listing entry. The name sits in the +// element titled with it, with the logo's aria-label as a fallback; the anchor +// text also carries the codes and the fleet size. +func airlineName(link *html.Node) string { + for _, attribute := range []string{"title", "aria-label"} { + element := findElement(link, func(node *html.Node) bool { return hasAttr(node, attribute) }) + + if element != nil { + return strings.TrimSpace(attr(element, attribute)) + } + } + return textContent(link) +} + +// leafTexts is the text of the `tag` elements under root that hold no other +// element. The name element is left out by its title attribute: an airline +// called "TAP" would otherwise read as an ICAO code. +func leafTexts(root *html.Node, tag string) []string { + var texts []string + + for _, element := range findElements(root, func(node *html.Node) bool { return node.Data == tag }) { + if hasAttr(element, "title") || findElement(element, func(*html.Node) bool { return true }) != nil { + continue + } + texts = append(texts, textContent(element)) + } + return texts +} + // countryToSlug slugifies a country name the way FR24 spells it in its data page // URLs, so feed rows can be matched against the Country constants. func countryToSlug(country string) string { @@ -214,49 +270,79 @@ func parseNumber(text string) *float64 { return &number } -// parseAirportsJSON parses the airports JSON feed into airports, keeping only -// the wanted countries when any are given. -func parseAirportsJSON(payload []byte, countries []Country) []*Airport { - decoder := json.NewDecoder(bytes.NewReader(bytes.TrimPrefix(payload, []byte("\xef\xbb\xbf")))) - - // Numbers stay as text, so a literal no float64 can hold is rejected - // instead of silently becoming an infinity. - decoder.UseNumber() +// parseInertiaPage returns the Inertia page object (component, props, version) +// of a FR24 data page, or false when the payload carries none. +// +// Served as HTML, the page keeps it JSON-encoded in the data-page attribute of +// its root element; asked with inertiaHeaders, FR24 sends it as the body, which +// the caller has already decoded into a map. Both shapes are accepted, so a +// test can feed the parser a decoded fixture the way the other ports do. +func parseInertiaPage(payload any) (map[string]any, bool) { + var page any + + switch typed := payload.(type) { + case map[string]any: + page = typed + case string: + return parseInertiaPage([]byte(typed)) + case []byte: + document, err := html.Parse(bytes.NewReader(typed)) - var data map[string]any + if err != nil { + log().Warn("parseInertiaPage: could not parse the response as HTML") + return nil, false + } - if err := decoder.Decode(&data); err != nil { - log().Warn("parseAirportsJSON: response is not valid JSON — FR24 feed may have changed") - return []*Airport{} - } + root := findElement(document, func(node *html.Node) bool { return hasAttr(node, "data-page") }) - // Decode stops at the first value, so a truncated or spliced body would - // otherwise pass as valid. The Python and Node.js parsers reject it. - if _, err := decoder.Token(); !errors.Is(err, io.EOF) { - log().Warn("parseAirportsJSON: response carries data past the JSON body — FR24 feed may have changed") - return []*Airport{} + if root == nil { + log().Warn("parseInertiaPage: no data-page attribute in response — FR24 page layout may have changed") + return nil, false + } + if err := json.Unmarshal([]byte(attr(root, "data-page")), &page); err != nil { + log().Warn("parseInertiaPage: data-page is not valid JSON — FR24 page layout may have changed") + return nil, false + } + default: + page = payload } - rows, ok := data["rows"].([]any) + object, ok := page.(map[string]any) if !ok { - log().Warn(`parseAirportsJSON: no "rows" array in response — FR24 feed may have changed`) - return []*Airport{} + log().Warn("parseInertiaPage: no page props in response — FR24 page layout may have changed") + return nil, false + } + if _, ok := object["props"].(map[string]any); !ok { + log().Warn("parseInertiaPage: no page props in response — FR24 page layout may have changed") + return nil, false } + return object, true +} - var wanted map[string]bool +// airportsCountry is one entry of the airports index: the country name as FR24 +// prints it ("UNITED STATES") and the slug of its listing page, which is also +// its Country constant. +type airportsCountry struct { + Name string + Slug string +} - if countries != nil { - wanted = make(map[string]bool, len(countries)) +// parseAirportsIndex parses the airports index (endpoints.airportsData) into +// its countries. The payload is the page HTML, or its decoded page object. +func parseAirportsIndex(payload any) []airportsCountry { + page, ok := parseInertiaPage(payload) + var rows []any - for _, country := range countries { - wanted[countryToSlug(string(country))] = true - } + if ok { + rows, _ = page["props"].(map[string]any)["airportsByCountry"].([]any) + } + if rows == nil { + log().Warn(`parseAirportsIndex: no "airportsByCountry" list in response — FR24 page layout may have changed`) + return []airportsCountry{} } - matched := make(map[string]bool, len(wanted)) - var unpositioned []string - airports := []*Airport{} + countries := []airportsCountry{} for _, entry := range rows { row, ok := entry.(map[string]any) @@ -265,58 +351,63 @@ func parseAirportsJSON(payload []byte, countries []Country) []*Airport { continue } - // Slugified only when filtering: this loop runs over every airport in - // the feed. - if wanted != nil { - slug := countryToSlug(toText(row["country"])) + name := toText(row["name"]) + slug := countryToSlug(name) - if !wanted[slug] { - continue - } - matched[slug] = true + if slug != "" { + countries = append(countries, airportsCountry{Name: name, Slug: slug}) } + } + return countries +} - latitude := toNumber(row["lat"]) - longitude := toNumber(row["lon"]) +// parseAirportsPage parses a country or state airports page into its airports +// and the paths of its state pages. The payload is the page HTML, or its +// decoded page object. +// +// A country with many airports lists its states instead, each with the path of +// its own page; those paths come back for the caller to fetch. The listing has +// no coordinates or altitude: [Client.GetAirport] has them for one airport. +func parseAirportsPage(payload any) (airports []*Airport, statePaths []string) { + airports, statePaths = []*Airport{}, []string{} + page, ok := parseInertiaPage(payload) - // One bad coordinate drops both: half a position reads as located. - if latitude == nil || longitude == nil { - latitude, longitude = nil, nil - unpositioned = append(unpositioned, toText(row["name"])) - } + if !ok { + return airports, statePaths + } + + props := page["props"].(map[string]any) + country, _ := props["country"].(map[string]any) + countryName := toText(country["name"]) + + rows, _ := props["airports"].([]any) + states, _ := props["states"].([]any) + for _, entry := range rows { + row, ok := entry.(map[string]any) + + if !ok { + continue + } airports = append(airports, newAirportFromBasicInfo(basicAirportInfo{ - Name: toText(row["name"]), - ICAO: toText(row["icao"]), - IATA: toText(row["iata"]), - Latitude: latitude, - Longitude: longitude, - Altitude: toNumber(row["alt"]), - Country: toText(row["country"]), + Name: toText(row["name"]), + ICAO: toText(row["icao"]), + IATA: toText(row["iata"]), + Country: countryName, })) } - // One line, not one per row: the feed carries every airport. - if len(unpositioned) > 0 { - log().Warn("parseAirportsJSON: airports with unusable coordinates carry no position", - "count", len(unpositioned), "examples", unpositioned[:min(3, len(unpositioned))]) - } - - if wanted != nil { - var missing []string + for _, entry := range states { + state, ok := entry.(map[string]any) - for slug := range wanted { - if !matched[slug] { - missing = append(missing, slug) - } + if !ok { + continue } - if len(missing) > 0 { - slices.Sort(missing) - log().Warn("parseAirportsJSON: no airports found for some countries — check the Country constants", - "countries", strings.Join(missing, ", ")) + if path := toText(state["url"]); path != "" { + statePaths = append(statePaths, path) } } - return airports + return airports, statePaths } // --- HTML helpers --- @@ -350,8 +441,15 @@ func attr(node *html.Node, name string) string { return "" } -func hasClass(node *html.Node, name string) bool { - return slices.Contains(strings.Fields(attr(node, "class")), name) +// hasAttr reports whether the element carries the attribute at all, which +// attr cannot tell from an empty value. +func hasAttr(node *html.Node, name string) bool { + for _, attribute := range node.Attr { + if attribute.Key == name { + return true + } + } + return false } // textContent concatenates the node's text, trimming each piece, as diff --git a/go/flightradarapi/parsers_test.go b/go/flightradarapi/parsers_test.go index 2bd0718..219dc3b 100644 --- a/go/flightradarapi/parsers_test.go +++ b/go/flightradarapi/parsers_test.go @@ -2,10 +2,9 @@ package flightradarapi import ( "encoding/json" - "fmt" - "math" "os" "path/filepath" + "slices" "strings" "testing" ) @@ -35,28 +34,6 @@ func airlinesByName(t *testing.T) map[string]Airline { return byName } -func airportsByIATA(t *testing.T, airports []*Airport) map[string]*Airport { - t.Helper() - byIATA := map[string]*Airport{} - - for _, airport := range airports { - byIATA[airport.IATA] = airport - } - return byIATA -} - -// airportsFromRow parses a one-row feed built from a raw JSON literal, the way -// both other ports do it. -func airportsFromRow(t *testing.T, row string) []*Airport { - t.Helper() - airports := parseAirportsJSON([]byte(`{"rows":[`+row+`]}`), nil) - - if len(airports) != 1 { - t.Fatalf("got %d airports for row %s, want 1", len(airports), row) - } - return airports -} - // --- parseAirlinesHTML --- func TestParseAirlinesHTMLExtractsKnownRows(t *testing.T) { @@ -102,289 +79,327 @@ func TestParseAirlinesHTMLParsesAircraftCount(t *testing.T) { for _, expected := range []struct { name string count int - }{{"LATAM Airlines", 340}, {"Gol", 140}} { + }{ + {"LATAM Airlines", 340}, + {"Gol", 140}, + // Thousands separator, as the page prints large fleets. + {"Delta Air Lines", 1900}, + } { count := byName[expected.name].NumAircrafts if count == nil || *count != expected.count { t.Errorf("%s: got %v aircraft, want %d", expected.name, count, expected.count) } } + + // Only the "Largest fleets" section carries a count. + if count := byName["Sky2"].NumAircrafts; count != nil { + t.Errorf("Sky2: got %d aircraft, want none", *count) + } +} + +func TestParseAirlinesHTMLMergesAnAirlineListedInTwoSections(t *testing.T) { + // Delta and Gol appear under "Largest fleets" and again under "All airlines". + seen := map[string]int{} + + for _, airline := range parseAirlinesHTML(loadFixture(t, "airlines.html")) { + seen[airline.Name]++ + } + for _, name := range []string{"Delta Air Lines", "Gol"} { + if seen[name] != 1 { + t.Errorf("%s: listed %d times, want once", name, seen[name]) + } + } +} + +func TestParseAirlinesHTMLKeepsACodeShapedName(t *testing.T) { + tap := airlinesByName(t)["TAP"] + + if tap.Name != "TAP" || tap.ICAO != "TAP" || tap.IATA != "TP" || tap.NumAircrafts != nil { + t.Errorf("got %+v, want TAP / TAP / TP with no count", tap) + } } func TestParseAirlinesHTMLSkipsInvalidRows(t *testing.T) { airlines := parseAirlinesHTML(loadFixture(t, "airlines.html")) - // 5 valid rows; 2 invalid (no notranslate, wrong href) must be skipped. - if len(airlines) != 5 { - t.Errorf("got %d airlines, want 5", len(airlines)) + // 6 airlines; the section link, the entry with no link, the wrong href and + // the one-letter name must be skipped. + if len(airlines) != 6 { + t.Errorf("got %d airlines, want 6", len(airlines)) + } + for _, airline := range airlines { + if airline.Name == "All" { + t.Errorf("the section link was read as an airline: %+v", airline) + } + } +} + +func TestParseAirlinesHTMLKeepsTheOrderOfThePage(t *testing.T) { + var names []string + + for _, airline := range parseAirlinesHTML(loadFixture(t, "airlines.html")) { + names = append(names, airline.Name) + } + + want := []string{"Delta Air Lines", "LATAM Airlines", "Gol", "Sky2", "SkyTeam", "TAP"} + + if !slices.Equal(names, want) { + t.Errorf("got %v, want %v", names, want) } } func TestParseAirlinesHTMLEmptyInputReturnsEmptyList(t *testing.T) { - for _, page := range []string{"", "

no tbody here

"} { + for _, page := range []string{"", "

no airlines here

"} { if airlines := parseAirlinesHTML([]byte(page)); len(airlines) != 0 { t.Errorf("got %d airlines for %q, want 0", len(airlines), page) } } } -// --- parseAirportsJSON --- +// --- parseInertiaPage --- -func TestParseAirportsJSONExtractsBasicFields(t *testing.T) { - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) +func decodeFixture(t *testing.T, name string) map[string]any { + t.Helper() + var page map[string]any - for _, code := range []string{"GRU", "GIG"} { - if _, ok := byIATA[code]; !ok { - t.Fatalf("missing airport %q", code) - } + if err := json.Unmarshal(loadFixture(t, name), &page); err != nil { + t.Fatalf("could not decode fixture %s: %v", name, err) } + return page +} - gru := byIATA["GRU"] +func TestParseInertiaPageReadsTheDataPageAttribute(t *testing.T) { + page, ok := parseInertiaPage(loadFixture(t, "airports_index.html")) - if gru.ICAO != "SBGR" || gru.Country != "Brazil" { - t.Errorf("GRU: got ICAO %q country %q", gru.ICAO, gru.Country) - } - if gru.Latitude == nil || math.Abs(*gru.Latitude-(-23.429991)) > 1e-6 { - t.Errorf("GRU latitude: got %v", gru.Latitude) + if !ok { + t.Fatal("no page object found") } - if gru.Longitude == nil || math.Abs(*gru.Longitude-(-46.4674)) > 1e-6 { - t.Errorf("GRU longitude: got %v", gru.Longitude) + if page["component"] != "Data/Airports" || page["version"] != "919b94d2a33f365a34f99828f6eeec6a" { + t.Errorf("got component %v version %v", page["component"], page["version"]) } - if gru.Altitude == nil || *gru.Altitude != 2436 { - t.Errorf("GRU altitude: got %v, want 2436", gru.Altitude) + if _, ok := page["props"].(map[string]any)["airportsByCountry"]; !ok { + t.Errorf("got props %v, want airportsByCountry among them", page["props"]) } } -func TestParseAirportsJSONFormatsAltitudeWithoutDecimals(t *testing.T) { - // Both other ports report 2436, never 2436.0. - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) +func TestParseInertiaPageAcceptsADecodedJSONBody(t *testing.T) { + body := decodeFixture(t, "airports_country.json") + page, ok := parseInertiaPage(body) - if got := formatNumber(*byIATA["GRU"].Altitude); got != "2436" { - t.Errorf("got altitude %q, want \"2436\"", got) + if !ok || page["component"] != body["component"] { + t.Errorf("got %v, want the body handed back", page) } } -func TestParseAirportsJSONKeepsEveryCountryWithoutFilter(t *testing.T) { - countries := map[string]bool{} - - for _, airport := range parseAirportsJSON(loadFixture(t, "airports.json"), nil) { - countries[airport.Country] = true +func TestParseInertiaPageAcceptsAString(t *testing.T) { + if _, ok := parseInertiaPage(string(loadFixture(t, "airports_index.html"))); !ok { + t.Error("the HTML as a string was not read") } +} - for _, country := range []string{"Brazil", "United States", "Spain"} { - if !countries[country] { - t.Errorf("missing country %q", country) +func TestParseInertiaPageRejectsUnrecognisedPayloads(t *testing.T) { + for _, payload := range []any{ + []byte(""), + []byte(`
`), + []byte(`
`), + []byte(`
`), + map[string]any{"component": "Data/Airports"}, + map[string]any{"props": "not an object"}, + []any{1, 2}, + nil, + 42, + } { + if page, ok := parseInertiaPage(payload); ok || page != nil { + t.Errorf("%v: got %v, want nothing", payload, page) } } } -func TestParseAirportsJSONFiltersByCountrySlug(t *testing.T) { - airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{"united-states"}) +// --- parseAirportsIndex --- + +func TestParseAirportsIndexListsEveryCountryWithItsSlug(t *testing.T) { + countries := parseAirportsIndex(loadFixture(t, "airports_index.html")) + var slugs []string - if len(airports) == 0 { - t.Fatal("no airports returned") + for _, country := range countries { + slugs = append(slugs, country.Slug) } - for _, airport := range airports { - if airport.Country != "United States" { - t.Errorf("got country %q, want United States", airport.Country) - } + if want := []string{"brazil", "myanmar-burma", "uruguay"}; !slices.Equal(slugs, want) { + t.Fatalf("got %v, want %v", slugs, want) + } + if countries[1].Name != "MYANMAR (BURMA)" { + t.Errorf("got name %q, want it as FR24 prints it", countries[1].Name) } } -func TestParseAirportsJSONAcceptsSeveralCountries(t *testing.T) { - airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{"brazil", "spain"}) - countries := map[string]bool{} +func TestParseAirportsIndexSlugsMatchTheCountryConstants(t *testing.T) { + known := map[Country]bool{} - for _, airport := range airports { - countries[airport.Country] = true + for _, country := range AllCountries() { + known[country] = true } - if len(countries) != 2 || !countries["Brazil"] || !countries["Spain"] { - t.Errorf("got countries %v, want Brazil and Spain", countries) + for _, country := range parseAirportsIndex(loadFixture(t, "airports_index.html")) { + if !known[Country(country.Slug)] { + t.Errorf("%q is not a Country constant", country.Slug) + } } } -func TestParseAirportsJSONInvalidCoordinatesBecomeNil(t *testing.T) { - // Regression: invalid coords used to be coerced to (0, 0), placing the - // airport in the Gulf of Guinea. - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) - bad := byIATA["BAD"] - - if bad == nil { - t.Fatal("missing airport BAD") - } - if bad.Latitude != nil || bad.Longitude != nil { - t.Errorf("got position (%v, %v), want no position", bad.Latitude, bad.Longitude) +func TestParseAirportsIndexInvalidPayloadReturnsEmptyList(t *testing.T) { + for _, payload := range []any{ + []byte(""), + map[string]any{"props": map[string]any{}}, + map[string]any{"props": map[string]any{"airportsByCountry": "x"}}, + map[string]any{"props": map[string]any{"airportsByCountry": []any{1, "x", map[string]any{"name": nil}}}}, + } { + if countries := parseAirportsIndex(payload); len(countries) != 0 { + t.Errorf("%v: got %v, want none", payload, countries) + } } } -func TestParseAirportsJSONRejectsNumericLookingJunk(t *testing.T) { - payload, err := json.Marshal(map[string]any{"rows": []any{map[string]any{ - "name": "Junk Airport", "iata": "JNK", "icao": "JJNK", - "lat": "43.30 N", "lon": -8.37725, "country": "Spain", "alt": "-1", - }}}) - - if err != nil { - t.Fatal(err) - } +// --- parseAirportsPage --- - airport := parseAirportsJSON(payload, nil)[0] +func iatas(airports []*Airport) []string { + codes := make([]string, 0, len(airports)) - // One bad coordinate drops both; altitude is independent and survives. - if airport.Latitude != nil || airport.Longitude != nil { - t.Errorf("got position (%v, %v), want no position", airport.Latitude, airport.Longitude) - } - if airport.Altitude == nil || *airport.Altitude != -1 { - t.Errorf("got altitude %v, want -1", airport.Altitude) + for _, airport := range airports { + codes = append(codes, airport.IATA) } + return codes } -func TestParseAirportsJSONAgreesWithOtherPortsOnUnicode(t *testing.T) { - cases := []struct { - name string - value string - expected *float64 - }{ - // Arabic-Indic digits, which a Unicode-aware \d matched before. - {"arabic-indic digits", "٤٣", nil}, - // U+FEFF, which JavaScript's trim() drops and Python's strip() keeps. - {"leading byte-order mark", "\ufeff43", nil}, - // U+001C, which Python's strip() drops and JavaScript's trim() keeps. - {"leading file separator", "\u001c43", nil}, - {"plain spaces are trimmed", " 43 ", float64Ptr(43)}, +func TestParseAirportsPageExtractsTheAirportsOfACountry(t *testing.T) { + airports, statePaths := parseAirportsPage(decodeFixture(t, "airports_country.json")) + + if len(statePaths) != 0 { + t.Errorf("got state paths %v, want none", statePaths) + } + if got := iatas(airports); !slices.Equal(got, []string{"CYR", "MVD"}) { + t.Fatalf("got %v, want CYR and MVD", got) } - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - row, err := json.Marshal(map[string]any{ - "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", - "lat": testCase.value, "lon": testCase.value, "alt": testCase.value, - }) + mvd := airports[1] - if err != nil { - t.Fatal(err) - } + if mvd.Name != "Montevideo Carrasco International Airport" || mvd.ICAO != "SUMU" || mvd.Country != "Uruguay" { + t.Errorf("got %q / %q / %q", mvd.Name, mvd.ICAO, mvd.Country) + } +} - airport := airportsFromRow(t, string(row))[0] +func TestParseAirportsPageListingCarriesNoPosition(t *testing.T) { + airports, _ := parseAirportsPage(decodeFixture(t, "airports_country.json")) - if !equalOptional(airport.Altitude, testCase.expected) { - t.Errorf("got altitude %v, want %v", airport.Altitude, testCase.expected) - } - }) + for _, airport := range airports { + if airport.Latitude != nil || airport.Longitude != nil || airport.Altitude != nil { + t.Errorf("%s: got position (%v, %v) altitude %v, want none", + airport.IATA, airport.Latitude, airport.Longitude, airport.Altitude) + } } } -func TestParseAirportsJSONNumericCoercion(t *testing.T) { - // Keep in step with the same list in the Python and Node.js suites. - cases := []struct { - name string - literal string - expected *float64 - }{ - {"whitespace", `" "`, nil}, - {"an array holding a number", `[43]`, nil}, - {"a coordinate with a hemisphere suffix", `"43.30 N"`, nil}, - {"an exponent overflowing a double", `"1e999"`, nil}, - {"400 plain digits", `"` + repeat("1", 400) + `"`, nil}, - {"a whole number as a string", `"2436"`, float64Ptr(2436)}, - {"a negative decimal", `-23.4`, float64Ptr(-23.4)}, - {"a genuine zero", `0`, float64Ptr(0)}, - } - - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - row := fmt.Sprintf( - `{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":%s,"lon":%s,"alt":%s}`, - testCase.literal, testCase.literal, testCase.literal) - airport := airportsFromRow(t, row)[0] - - for label, got := range map[string]*float64{ - "latitude": airport.Latitude, - "longitude": airport.Longitude, - "altitude": airport.Altitude, - } { - if !equalOptional(got, testCase.expected) { - t.Errorf("%s: got %v, want %v", label, got, testCase.expected) - } - } - }) - } -} - -func TestParseAirportsJSONSurvivesRawNumbersNoDoubleCanHold(t *testing.T) { - for _, literal := range []string{repeat("1", 400), "9007199254740993", "-" + repeat("1", 400)} { - row := fmt.Sprintf( - `{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":%s,"lon":1,"alt":2}`, literal) - latitude := airportsFromRow(t, row)[0].Latitude - - if latitude != nil && (math.IsInf(*latitude, 0) || math.IsNaN(*latitude)) { - t.Errorf("%s: got %v, want nil or a finite number", literal, *latitude) - } +func TestParseAirportsPageReturnsTheStatePagesOfASplitCountry(t *testing.T) { + airports, statePaths := parseAirportsPage(decodeFixture(t, "airports_country_states.json")) + + if len(airports) != 0 { + t.Errorf("got %d airports, want none", len(airports)) + } + if want := []string{"/data/airports/brazil/rj", "/data/airports/brazil/sp"}; !slices.Equal(statePaths, want) { + t.Errorf("got %v, want %v", statePaths, want) } } -func TestParseAirportsJSONKeepsTextFieldsAsStrings(t *testing.T) { - // Anything else breaks GetCountryFlag(airport.Country). - for _, literal := range []string{"null", "0", "false", "true", "[]", "{}", "123"} { - row := fmt.Sprintf( - `{"name":%s,"iata":%s,"icao":%s,"country":%s,"lat":1,"lon":2,"alt":3}`, - literal, literal, literal, literal) - airport := airportsFromRow(t, row)[0] - - if airport.Name != "" || airport.IATA != "" || airport.ICAO != "" || airport.Country != "" { - t.Errorf("%s: got %q/%q/%q/%q, want empty strings", - literal, airport.Name, airport.IATA, airport.ICAO, airport.Country) +func TestParseAirportsPageStatePageKeepsTheCountryName(t *testing.T) { + airports, statePaths := parseAirportsPage(decodeFixture(t, "airports_state_sp.json")) + + if len(statePaths) != 0 { + t.Errorf("got state paths %v, want none", statePaths) + } + if got := iatas(airports); !slices.Equal(got, []string{"GRU", "CGH"}) { + t.Errorf("got %v, want GRU and CGH", got) + } + for _, airport := range airports { + if airport.Country != "Brazil" { + t.Errorf("%s: got country %q, want Brazil", airport.IATA, airport.Country) } } } -func TestParseAirportsJSONCountrySpellingAndSlugRoundTrip(t *testing.T) { - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) - ann := byIATA["VBA"] +func TestParseAirportsPageReadsAnHTMLPage(t *testing.T) { + airports, _ := parseAirportsPage(loadFixture(t, "airports_country.html")) - if ann == nil { - t.Fatal("missing airport VBA") + if got := iatas(airports); !slices.Equal(got, []string{"RGN"}) { + t.Fatalf("got %v, want RGN", got) } - if ann.Country != "Myanmar (Burma)" { - t.Errorf("got country %q", ann.Country) + // FR24 spells it "Myanmar (Burma)" on the page and "myanmar-burma" in URLs. + if airports[0].Country != "Myanmar (Burma)" { + t.Errorf("got country %q", airports[0].Country) } - if slug := countryToSlug(ann.Country); slug != "myanmar-burma" { - t.Errorf("got slug %q, want myanmar-burma", slug) - } - if ann.Altitude == nil || *ann.Altitude != 43 { - t.Errorf("got altitude %v, want 43", ann.Altitude) + if slug := countryToSlug(airports[0].Country); slug != string(CountryMyanmarBurma) { + t.Errorf("got slug %q, want %q", slug, CountryMyanmarBurma) } } -func TestParseAirportsJSONFilterAcceptsEitherCountrySpelling(t *testing.T) { - for _, spelling := range []Country{"myanmar-burma", "Myanmar (Burma)", CountryMyanmarBurma} { - airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{spelling}) +func TestParseAirportsPageSkipsRowsThatAreNotObjects(t *testing.T) { + payload := map[string]any{"props": map[string]any{ + "country": map[string]any{"name": "Nowhere"}, + "airports": []any{1, "x", nil, map[string]any{"iata": "RRR"}}, + }} + airports, _ := parseAirportsPage(payload) - if len(airports) != 1 { - t.Errorf("%q: got %d airports, want 1", spelling, len(airports)) - } + if got := iatas(airports); !slices.Equal(got, []string{"RRR"}) { + t.Errorf("got %v, want only RRR", got) } } -func TestParseAirportsJSONSkipsRowsThatAreNotObjects(t *testing.T) { - payload := []byte(`{"rows":[[1,2,3],"x",7,null,` + - `{"name":"Real","iata":"RRR","icao":"RRRR","country":"Spain","lat":1,"lon":2,"alt":3}]}`) - airports := parseAirportsJSON(payload, nil) +func TestParseAirportsPageKeepsTextFieldsAsStrings(t *testing.T) { + // Anything else breaks GetCountryFlag(airport.Country). + payload := map[string]any{"props": map[string]any{ + "country": map[string]any{"name": []any{"Brazil"}}, + "airports": []any{map[string]any{"name": 42.0, "iata": nil, "icao": map[string]any{"x": 1.0}}}, + }} + airports, _ := parseAirportsPage(payload) + + if len(airports) != 1 { + t.Fatalf("got %d airports, want 1", len(airports)) + } + + airport := airports[0] - if len(airports) != 1 || airports[0].IATA != "RRR" { - t.Errorf("got %d airports, want only RRR", len(airports)) + if airport.Name != "" || airport.IATA != "" || airport.ICAO != "" || airport.Country != "" { + t.Errorf("got %q/%q/%q/%q, want empty strings", airport.Name, airport.IATA, airport.ICAO, airport.Country) } } -func TestParseAirportsJSONUnknownCountryReturnsEmptyList(t *testing.T) { - if airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{"atlantis"}); len(airports) != 0 { - t.Errorf("got %d airports, want 0", len(airports)) +func TestParseAirportsPageIgnoresMalformedStates(t *testing.T) { + payload := map[string]any{"props": map[string]any{"states": []any{ + map[string]any{"url": "/data/airports/x/a"}, + map[string]any{"url": ""}, + map[string]any{"code": "B"}, + "junk", + map[string]any{"url": 3.0}, + }}} + airports, statePaths := parseAirportsPage(payload) + + if len(airports) != 0 || !slices.Equal(statePaths, []string{"/data/airports/x/a"}) { + t.Errorf("got %d airports and state paths %v, want only /data/airports/x/a", len(airports), statePaths) } } -func TestParseAirportsJSONInvalidPayloadReturnsEmptyList(t *testing.T) { - for _, payload := range []string{"", "{}", "not json"} { - if airports := parseAirportsJSON([]byte(payload), nil); len(airports) != 0 { - t.Errorf("%q: got %d airports, want 0", payload, len(airports)) +func TestParseAirportsPageInvalidPayloadReturnsNothing(t *testing.T) { + for _, payload := range []any{ + []byte(""), + []byte("not a data page"), + map[string]any{"props": map[string]any{}}, + map[string]any{"props": map[string]any{"country": "Brazil", "airports": "none", "states": 7.0}}, + } { + airports, statePaths := parseAirportsPage(payload) + + if airports == nil || statePaths == nil { + t.Errorf("%v: got nil slices, want empty ones", payload) + } + if len(airports) != 0 || len(statePaths) != 0 { + t.Errorf("%v: got %d airports and %v, want nothing", payload, len(airports), statePaths) } } } @@ -431,46 +446,12 @@ func TestCountryConstantsRoundTripThroughTheSlugifier(t *testing.T) { } } -func equalOptional(got, want *float64) bool { - if got == nil || want == nil { - return got == nil && want == nil - } - return *got == *want -} - // float64Ptr is the shorthand the table-driven tests need for optional numbers. func float64Ptr(value float64) *float64 { return &value } -func repeat(text string, times int) string { - result := make([]byte, 0, len(text)*times) - - for range times { - result = append(result, text...) - } - return string(result) -} - -func TestParseAirportsJSONRejectsDataPastTheJSONBody(t *testing.T) { - // Decode reads the first value only, so a spliced body used to pass. - payload := []byte(`{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",` + - `"lat":1,"lon":2,"alt":3}]}trailing garbage`) - - if airports := parseAirportsJSON(payload, nil); len(airports) != 0 { - t.Errorf("got %d airports, want the body rejected", len(airports)) - } - - // Trailing whitespace is not garbage. - clean := []byte(`{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",` + - `"lat":1,"lon":2,"alt":3}]}` + "\n\n ") - - if airports := parseAirportsJSON(clean, nil); len(airports) != 1 { - t.Errorf("got %d airports, want the body accepted", len(airports)) - } -} - func TestTextFieldsReadAPayloadDecodedWithUseNumber(t *testing.T) { - // This package decodes the airports feed that way, and the exported - // constructors take a map the caller may have decoded the same way. + // The exported constructors take a map the caller may have decoded with + // UseNumber, and must read a number there as text. decoder := json.NewDecoder(strings.NewReader(`{"age": 12, "name": "Lukla"}`)) decoder.UseNumber() diff --git a/go/flightradarapi/testdata/airlines.html b/go/flightradarapi/testdata/airlines.html index 3c56b99..020dfd5 100644 --- a/go/flightradarapi/testdata/airlines.html +++ b/go/flightradarapi/testdata/airlines.html @@ -1,60 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- LATAM Airlines - xxLA / LAN340 aircraft
- Gol - xxG3 / GLO140 aircraft
- Delta Air Lines - xxDL / DAL900 aircraft
- Sky2 - xxSK10 aircraft
- SkyTeam - xxSKT15 aircraft
placeholder
- Should Skip -
- + + +Airlines - Flightradar24 + +
+
+

Largest fleets

+ All +
+ +
+
+

All airlines

+ +
+ + diff --git a/go/flightradarapi/testdata/airports.json b/go/flightradarapi/testdata/airports.json deleted file mode 100644 index 924f9db..0000000 --- a/go/flightradarapi/testdata/airports.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1786010717", - "rows": [ - { - "name": "Sao Paulo Guarulhos International Airport", - "iata": "GRU", - "icao": "SBGR", - "lat": -23.429991, - "lon": -46.4674, - "country": "Brazil", - "alt": 2436 - }, - { - "name": "Rio de Janeiro Galeao International Airport", - "iata": "GIG", - "icao": "SBGL", - "lat": -22.805696, - "lon": -43.25523, - "country": "Brazil", - "alt": 22 - }, - { - "name": "Sao Paulo Congonhas Airport", - "iata": "CGH", - "icao": "SBSP", - "lat": -23.627348, - "lon": -46.655987, - "country": "Brazil", - "alt": 2614 - }, - { - "name": "Los Angeles International Airport", - "iata": "LAX", - "icao": "KLAX", - "lat": 33.94252, - "lon": -118.406998, - "country": "United States", - "alt": 125 - }, - { - "name": "New York John F. Kennedy International Airport", - "iata": "JFK", - "icao": "KJFK", - "lat": 40.639751, - "lon": -73.7789, - "country": "United States", - "alt": 14 - }, - { - "name": "A Coruna Airport", - "iata": "LCG", - "icao": "LECO", - "lat": 43.302059, - "lon": -8.37725, - "country": "Spain", - "alt": 326 - }, - { - "name": "Ann Airport", - "iata": "VBA", - "icao": "VYAN", - "lat": 19.770643, - "lon": 94.026405, - "country": "Myanmar (Burma)", - "alt": "43" - }, - { - "name": "Broken Coords Airport", - "iata": "BAD", - "icao": "SBAD", - "lat": "n/a", - "lon": "", - "country": "Brazil", - "alt": 0 - } - ] -} diff --git a/go/flightradarapi/testdata/airports_country.html b/go/flightradarapi/testdata/airports_country.html new file mode 100644 index 0000000..833d4f4 --- /dev/null +++ b/go/flightradarapi/testdata/airports_country.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/go/flightradarapi/testdata/airports_country.json b/go/flightradarapi/testdata/airports_country.json new file mode 100644 index 0000000..09b4811 --- /dev/null +++ b/go/flightradarapi/testdata/airports_country.json @@ -0,0 +1,51 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "uruguay" + }, + "query": [] + }, + "country": { + "id": 238, + "name": "Uruguay", + "slug": "uruguay" + }, + "airports": [ + { + "id": 787, + "name": "Colonia Laguna de Los Patos International Airport", + "city": "Colonia", + "iata": "CYR", + "icao": "SUCA", + "total": 8 + }, + { + "id": 2349, + "name": "Montevideo Carrasco International Airport", + "city": "Montevideo", + "iata": "MVD", + "icao": "SUMU", + "total": 632 + } + ], + "airportImages": { + "SUMU": { + "thumbnail": "https://cdn.jetphotos.com/200/5/1.jpg", + "copyright": "Someone" + } + }, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/uruguay", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/flightradarapi/testdata/airports_country_states.json b/go/flightradarapi/testdata/airports_country_states.json new file mode 100644 index 0000000..ff8bca1 --- /dev/null +++ b/go/flightradarapi/testdata/airports_country_states.json @@ -0,0 +1,41 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "brazil" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "states": [ + { + "code": "RJ", + "name": "Rio de Janeiro", + "total": 1, + "url": "/data/airports/brazil/rj" + }, + { + "code": "SP", + "name": "Sao Paulo", + "total": 2, + "url": "/data/airports/brazil/sp" + } + ], + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/flightradarapi/testdata/airports_index.html b/go/flightradarapi/testdata/airports_index.html new file mode 100644 index 0000000..a55ed2f --- /dev/null +++ b/go/flightradarapi/testdata/airports_index.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/go/flightradarapi/testdata/airports_state_rj.json b/go/flightradarapi/testdata/airports_state_rj.json new file mode 100644 index 0000000..77d7a6b --- /dev/null +++ b/go/flightradarapi/testdata/airports_state_rj.json @@ -0,0 +1,44 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "rj" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "RJ", + "name": "Rio de Janeiro", + "slug": "rj" + }, + "airports": [ + { + "id": 1252, + "name": "Rio de Janeiro Galeao International Airport", + "city": "Rio de Janeiro", + "iata": "GIG", + "icao": "SBGL", + "total": 2500 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/rj", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/flightradarapi/testdata/airports_state_sp.json b/go/flightradarapi/testdata/airports_state_sp.json new file mode 100644 index 0000000..46c60c6 --- /dev/null +++ b/go/flightradarapi/testdata/airports_state_sp.json @@ -0,0 +1,52 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "sp" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "SP", + "name": "Sao Paulo", + "slug": "sp" + }, + "airports": [ + { + "id": 1255, + "name": "Sao Paulo Guarulhos International Airport", + "city": "Sao Paulo", + "iata": "GRU", + "icao": "SBGR", + "total": 6796 + }, + { + "id": 1254, + "name": "Sao Paulo Congonhas Airport", + "city": "Sao Paulo", + "iata": "CGH", + "icao": "SBSP", + "total": 3100 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/sp", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/go.mod b/go/go.mod index 79d19fc..746c70a 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,7 +3,7 @@ module github.com/JeanExtreme002/FlightRadarAPI/go go 1.26.0 require ( - github.com/andybalholm/brotli v1.2.2 + github.com/andybalholm/brotli v1.2.7 golang.org/x/net v0.62.0 golang.org/x/text v0.43.0 ) diff --git a/go/go.sum b/go/go.sum index 45956a4..4df38b2 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,5 +1,5 @@ -github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= -github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A= +github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= diff --git a/nodejs/FlightRadarAPI/api.js b/nodejs/FlightRadarAPI/api.js index 6d50ebf..00eb664 100644 --- a/nodejs/FlightRadarAPI/api.js +++ b/nodejs/FlightRadarAPI/api.js @@ -5,7 +5,7 @@ const Flight = require("./entities/flight"); const FlightTrackerConfig = require("./flightTrackerConfig"); const { AirportNotFoundError, LoginError } = require("./errors"); const { isNumeric, radians, rad2deg } = require("./util"); -const { parseAirlinesHtml, parseAirportsJson, countryToSlug } = require("./parsers"); +const { parseAirlinesHtml, parseAirportsIndex, parseAirportsPage, parseInertiaPage, countryToSlug } = require("./parsers"); /** @@ -200,8 +200,14 @@ class FlightRadar24API { /** * Return a list with all airports, optionally narrowed to some countries. * + * FR24 lists airports one page per country, and one per state for the + * largest countries, so this fetches one page per country asked for (a few + * hundred for every airport), `maxWorkers` at a time. The listing carries + * the name, codes and country only: use getAirport(code) for the position + * and altitude of an airport. + * * @param {Iterable|string} [countries] - Country names from the Countries enum, - * as any iterable or a single value; every country when omitted + * or their slug strings, as any iterable or a single value; every country when omitted * @return {Promise>} */ async getAirports(countries) { @@ -222,10 +228,94 @@ class FlightRadar24API { if (wanted && wanted.length === 0) return []; + // The index names every country, and declares the asset version the + // JSON requests below have to quote. const { content } = await this.__client.request( - Core.airportsJsonUrl, { headers: Core.jsonHeaders, timeout: this.timeout }, + Core.airportsDataUrl, { headers: Core.htmlHeaders, timeout: this.timeout }, + ); + const index = parseInertiaPage(content); + const version = typeof index?.["version"] === "string" ? index["version"] : null; + + const slugs = wanted === null ? + parseAirportsIndex(index ?? {}).map((country) => country.slug) : + wanted.map(countryToSlug); + const uniqueSlugs = [...new Set(slugs.filter((slug) => slug))]; + + const countryPages = await this.__getAirportPages(uniqueSlugs.map(Core.airportsByCountryUrl), version); + + // Second pass for the countries split by state, every state at once. + const stateOwners = []; + const stateUrls = []; + + countryPages.forEach(({ statePaths }, owner) => { + for (const path of statePaths) { + stateOwners.push(owner); + stateUrls.push(path.startsWith("http") ? path : Core.flightRadarBaseUrl + path); + } + }); + + const statePages = await this.__getAirportPages(stateUrls, version); + + // Grouped by country, in the order asked for, each country's states after it. + const airportsByCountry = countryPages.map(({ airports }) => [...airports]); + + statePages.forEach(({ airports }, i) => airportsByCountry[stateOwners[i]].push(...airports)); + + // A country FR24 does not know answers with the index page, which lists + // neither airports nor states. + const missing = uniqueSlugs.filter( + (slug, i) => countryPages[i].airports.length === 0 && countryPages[i].statePaths.length === 0, + ); + + if (missing.length > 0) { + console.warn(`getAirports: no airports found for ${missing.join(", ")} — check the Countries enum.`); + } + + return airportsByCountry.flat(); + } + + /** + * Fetch and parse the airport listing pages at `urls`, in order, maxWorkers at a time. + * + * @param {Array} urls + * @param {string|null} version - Asset version declared by the index page + * @return {Promise, statePaths: Array}>>} + */ + async __getAirportPages(urls, version) { + const pages = new Array(urls.length); + + await mapConcurrent(urls.map((url, i) => [url, i]), this.maxWorkers, async ([url, i]) => { + pages[i] = parseAirportsPage(await this.__getInertiaPage(url, version)); + }); + + return pages; + } + + /** + * Fetch a FR24 data page as its Inertia page object, falling back to the HTML. + * + * With the asset version FR24 answers the bare page JSON, a fraction of the + * HTML. A 409 means the assets were redeployed since the version was read: + * the HTML carries the same page object and needs no version. + * + * @param {string} url + * @param {string|null} version + * @return {Promise} + */ + async __getInertiaPage(url, version) { + if (version !== null) { + const headers = { ...Core.inertiaHeaders, "x-inertia-version": version }; + const { content, statusCode } = await this.__client.requestStandalone( + url, { headers, timeout: this.timeout, allowedErrorCodes: [409] }, + ); + + if (statusCode !== 409) return content; + } + + const { content } = await this.__client.requestStandalone( + url, { headers: Core.htmlHeaders, timeout: this.timeout }, ); - return parseAirportsJson(content, wanted); + return content; } /** diff --git a/nodejs/FlightRadarAPI/core.js b/nodejs/FlightRadarAPI/core.js index 638362c..ae8f6f6 100644 --- a/nodejs/FlightRadarAPI/core.js +++ b/nodejs/FlightRadarAPI/core.js @@ -46,9 +46,12 @@ const Core = { apiAirportDataUrl: `${API_FR24_BASE}/airport.json`, airportDataUrl: (code) => `${FR24_BASE}/airports/traffic-stats/?airport=${code}`, + // The listing is split by country, and by state for the largest countries. + // These are Inertia pages: the data rides JSON-encoded in the data-page + // attribute of the HTML, or comes as the whole body when the request carries + // `inertiaHeaders` and the asset version the index page declares. airportsDataUrl: `${FR24_BASE}/data/airports`, - // Every airport in one payload; the per-country pages above are client-rendered. - airportsJsonUrl: `${FR24_BASE}/_json/airports.php`, + airportsByCountryUrl: (slug) => `${FR24_BASE}/data/airports/${slug}`, airlinesDataUrl: `${FR24_BASE}/data/airlines`, @@ -72,6 +75,11 @@ const Core = { headers: baseHeaders, jsonHeaders: { accept: "application/json", ...baseHeaders }, imageHeaders: { accept: "image/gif, image/jpg, image/jpeg, image/png", ...baseHeaders }, + // Asks an Inertia page for its page object instead of the HTML around it. + // `x-inertia-version` must match the deployed assets or FR24 answers 409. + inertiaHeaders: { + "accept": "application/json", ...baseHeaders, "x-inertia": "true", "x-requested-with": "XMLHttpRequest", + }, htmlHeaders: { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "accept-encoding": "gzip, deflate, br", diff --git a/nodejs/FlightRadarAPI/index.d.ts b/nodejs/FlightRadarAPI/index.d.ts index 533d408..1af0f81 100644 --- a/nodejs/FlightRadarAPI/index.d.ts +++ b/nodejs/FlightRadarAPI/index.d.ts @@ -116,8 +116,11 @@ export class FlightRadar24API { /** * Return a list with all airports, optionally narrowed to some countries. * - * @param {Iterable|string} [countries] - Country names, as any iterable - * or a single value; every country when omitted + * The listing carries the name, codes and country only: latitude, longitude + * and altitude are null. Use getAirport(code) for the position of an airport. + * + * @param {Iterable|string} [countries] - Country names from the Countries enum, + * or their slug strings, as any iterable or a single value; every country when omitted */ getAirports(countries?: Iterable | string): Promise; diff --git a/nodejs/FlightRadarAPI/parsers.js b/nodejs/FlightRadarAPI/parsers.js index 79a16dd..2e0c396 100644 --- a/nodejs/FlightRadarAPI/parsers.js +++ b/nodejs/FlightRadarAPI/parsers.js @@ -1,73 +1,114 @@ const { parse } = require("node-html-parser"); const Airport = require("./entities/airport"); +// An airline code badge: a two-letter IATA code or a three-letter ICAO code. +// ASCII only, so a stray Unicode letter does not pass for a code. +const AIRLINE_CODE_PATTERN = /^[A-Z0-9]{2,3}$/; + +// A fleet size as the page prints it: "340" or "3,878". +const AIRCRAFT_COUNT_PATTERN = /^\d{1,3}(?:[,.]\d{3})*$|^\d+$/; + +const AIRLINES_PATH_PREFIX = "/data/airlines/"; + +/** + * Whether an element holds no other element (text and comments only). + * + * @param {HTMLElement} element + * @return {boolean} + */ +function isLeaf(element) { + return element.childNodes.every((node) => node.nodeType !== 1); +} + +/** + * Text of the `tag` elements under `root` that hold no other element. + * + * The name element is left out by its `title` attribute: an airline called + * "TAP" would otherwise read as an ICAO code. + * + * @param {HTMLElement} root + * @param {string} tag + * @return {Array} + */ +function leafTexts(root, tag) { + return root.querySelectorAll(tag) + .filter((element) => isLeaf(element) && !element.hasAttribute("title")) + .map((element) => element.text.trim()); +} + +/** + * The airline name of a listing entry. + * + * The name sits in the element titled with it, with the logo's aria-label as a + * fallback; the anchor text also carries the codes and the fleet size. + * + * @param {HTMLElement} link + * @return {string} + */ +function airlineName(link) { + for (const attribute of ["title", "aria-label"]) { + const element = link.querySelector(`[${attribute}]`); + + if (element) return String(element.getAttribute(attribute) ?? "").trim(); + } + + return link.text.trim(); +} + /** - * Parse the airlines listing HTML page into a list of airline objects. + * Parse the airlines listing page into a list of airline objects. + * + * Every airline is a link to its data page, with the IATA and ICAO codes as + * badges next to the name. The "Largest fleets" section repeats a few airlines + * with their aircraft count; an airline the page lists twice becomes one entry + * carrying everything the page said about it. * * @param {string|Buffer} html * @return {Array} */ function parseAirlinesHtml(html) { - const root = parse(typeof html === "string" ? html : html.toString()); - const tbody = root.querySelector("tbody"); + const root = parse(toText(html) ?? ""); + const links = root.querySelectorAll("a[href]") + .filter((link) => String(link.getAttribute("href")).startsWith(AIRLINES_PATH_PREFIX)); - if (!tbody) { - console.warn("parseAirlinesHtml: no in response — FR24 page layout may have changed."); + if (links.length === 0) { + console.warn("parseAirlinesHtml: no airline links in response — FR24 page layout may have changed."); return []; } - const airlines = []; - - for (const tr of tbody.querySelectorAll("tr")) { - const tdNotranslate = tr.querySelector("td.notranslate"); - - if (!tdNotranslate) continue; + // Keyed by href, so an airline listed in two sections becomes one entry. + const airlines = new Map(); - const aElement = tdNotranslate.querySelector("a[href^='/data/airlines']"); + for (const link of links) { + const name = airlineName(link); - if (!aElement) continue; + if (name.length < 2) continue; - const airlineName = aElement.text.trim(); + const codes = leafTexts(link, "span").filter((text) => AIRLINE_CODE_PATTERN.test(text)); + const iata = codes.find((code) => code.length === 2) ?? null; + const icao = codes.find((code) => code.length === 3) ?? null; - if (airlineName.length < 2) continue; + // Section links ("All") point under /data/airlines/ too; a code tells + // an airline apart from them. + if (iata === null && icao === null) continue; - const tdElements = tr.querySelectorAll("td"); - let iata = null; - let icao = null; + const countText = leafTexts(link, "strong").find((text) => AIRCRAFT_COUNT_PATTERN.test(text)); + const nAircrafts = countText === undefined ? null : parseInt(countText.replace(/\D/g, ""), 10); - if (tdElements.length >= 4) { - const codesText = tdElements[3].text.trim(); + const href = link.getAttribute("href"); - if (codesText.includes(" / ")) { - const parts = codesText.split(" / "); - - if (parts.length === 2) { - iata = parts[0].trim(); - icao = parts[1].trim(); - } - } - else if (codesText.length === 2) { - iata = codesText; - } - else if (codesText.length === 3) { - icao = codesText; - } + if (!airlines.has(href)) { + airlines.set(href, { "Name": name, "ICAO": null, "IATA": null, "n_aircrafts": null }); } - let nAircrafts = null; - - if (tdElements.length >= 5) { - const aircraftsText = tdElements[4].text.trim(); + const airline = airlines.get(href); - if (aircraftsText) { - nAircrafts = parseInt(aircraftsText.split(" ")[0].trim(), 10); - } + for (const [key, value] of [["ICAO", icao], ["IATA", iata], ["n_aircrafts", nAircrafts]]) { + if (airline[key] === null) airline[key] = value; } - - airlines.push({ "Name": airlineName, "ICAO": icao, "IATA": iata, "n_aircrafts": nAircrafts }); } - return airlines; + return [...airlines.values()]; } /** @@ -82,35 +123,10 @@ function countryToSlug(country) { // Punctuation becomes a hyphen rather than being deleted: FR24's own assets // are named that way, e.g. flags-small/cote-d-ivoire.svg. return String(country ?? "") - .normalize("NFKD").replace(/[\u0300-\u036f]/g, "") + .normalize("NFKD").replace(/[̀-ͯ]/g, "") .toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, ""); } -// ASCII decimals only, so both ports accept and reject the same strings. -const NUMERIC_PATTERN = /^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?$/; - -// trim() and str.strip() disagree on U+FEFF and U+001C-U+001F: pick one set. -const SURROUNDING_SPACE = /^[ \t\n\r\f\v]+|[ \t\n\r\f\v]+$/g; - -/** - * Coerce a numeric feed field into a number, or null when it is unusable. - * - * An unusable coordinate must not become 0: that would place the airport in the - * Gulf of Guinea instead of marking its position as unknown. - * - * @param {*} value - * @return {number|null} - */ -function toNumber(value) { - if (typeof value === "number") return Number.isFinite(value) ? value : null; - // Not stringified: String([43]) would pass as 43. - if (typeof value !== "string") return null; - - const text = value.replace(SURROUNDING_SPACE, ""); - - return NUMERIC_PATTERN.test(text) ? toNumber(Number(text)) : null; -} - /** * Keep a text field as a string, or "" when the feed sends anything else. * @@ -144,90 +160,127 @@ function toText(payload) { } /** - * Parse the airports JSON feed into a list of Airport instances. + * Whether a value is a plain object: arrays are objects in JS, and Python's + * isinstance(value, dict) drops them. + * + * @param {*} value + * @return {boolean} + */ +function isObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** + * Return the Inertia page object (component, props, version) of a FR24 data page. + * + * Served as HTML, the page keeps it JSON-encoded in the data-page attribute of + * its root element; asked with Core.inertiaHeaders, FR24 sends it as the body, + * which `request()` has already decoded into an object. * - * @param {object|string|Buffer} payload - Body of Core.airportsJsonUrl - * @param {Array} [countries] - Country slugs from the Countries enum; every airport is kept when omitted - * @return {Array} + * @param {object|string|Buffer|ArrayBuffer} payload + * @return {object|null} */ -function parseAirportsJson(payload, countries = null) { - let data = payload; +function parseInertiaPage(payload) { + let page = payload; const text = toText(payload); if (text !== null) { + // node-html-parser decodes the attribute's entities (", '...). + const root = parse(text).querySelector("[data-page]"); + + if (!root) { + console.warn("parseInertiaPage: no data-page attribute in response — FR24 page layout may have changed."); + return null; + } + try { - data = JSON.parse(text); + page = JSON.parse(root.getAttribute("data-page")); } catch { - console.warn("parseAirportsJson: response is not valid JSON — FR24 feed may have changed."); - return []; + console.warn("parseInertiaPage: data-page is not valid JSON — FR24 page layout may have changed."); + return null; } } - const rows = data && Array.isArray(data["rows"]) ? data["rows"] : null; + if (!isObject(page) || !isObject(page["props"])) { + console.warn("parseInertiaPage: no page props in response — FR24 page layout may have changed."); + return null; + } + + return page; +} - if (!rows) { - console.warn("parseAirportsJson: no \"rows\" array in response — FR24 feed may have changed."); +/** + * Parse the airports index (Core.airportsDataUrl) into its countries. + * + * Each entry has the country "name" as FR24 prints it ("UNITED STATES") and the + * "slug" of its listing page, which is also its Countries enum value. + * + * @param {object|string|Buffer|ArrayBuffer} payload + * @return {Array<{name: string, slug: string}>} + */ +function parseAirportsIndex(payload) { + const page = parseInertiaPage(payload); + const rows = page !== null ? page["props"]["airportsByCountry"] : null; + + if (!Array.isArray(rows)) { + console.warn("parseAirportsIndex: no \"airportsByCountry\" list in response — FR24 page layout may have changed."); return []; } - const wanted = countries ? new Set(countries.map(countryToSlug)) : null; - const matched = new Set(); - const unpositioned = []; - const airports = []; + const countries = []; for (const row of rows) { - // Arrays are objects in JS; Python's isinstance(row, dict) drops them. - if (!row || typeof row !== "object" || Array.isArray(row)) continue; + if (!isObject(row)) continue; - // Slugified only when filtering: it is otherwise unused, and this loop - // runs over every airport in the feed. - if (wanted) { - const slug = countryToSlug(row["country"]); + const name = toStringField(row["name"]); + const slug = countryToSlug(name); - if (!wanted.has(slug)) continue; - matched.add(slug); - } + if (slug) countries.push({ name, slug }); + } - let latitude = toNumber(row["lat"]); - let longitude = toNumber(row["lon"]); + return countries; +} - // One bad coordinate drops both: half a position reads as located. - if (latitude === null || longitude === null) { - latitude = null; - longitude = null; - unpositioned.push(toStringField(row["name"])); - } +/** + * Parse a country or state airports page into its airports and state pages. + * + * A country with many airports lists its states instead, each with the path of + * its own page; those paths come back for the caller to fetch. The listing has + * no coordinates or altitude: `getAirport(code)` has them for one airport. + * + * @param {object|string|Buffer|ArrayBuffer} payload + * @return {{airports: Array, statePaths: Array}} + */ +function parseAirportsPage(payload) { + const page = parseInertiaPage(payload); - airports.push(new Airport({ - "name": toStringField(row["name"]), - "icao": toStringField(row["icao"]), - "iata": toStringField(row["iata"]), - "lat": latitude, - "lon": longitude, - "alt": toNumber(row["alt"]), - "country": toStringField(row["country"]), - })); + if (page === null) { + return { airports: [], statePaths: [] }; } - // One line, not one per row: the feed carries every airport. - if (unpositioned.length > 0) { - const sample = unpositioned.slice(0, 3).map((name) => `"${name}"`).join(", "); - console.warn( - `parseAirportsJson: ${unpositioned.length} airport(s) had unusable coordinates ` + - `and carry no position (e.g. ${sample}).`, - ); - } + const props = page["props"]; + const country = props["country"]; + const countryName = isObject(country) ? toStringField(country["name"]) : ""; - if (wanted) { - const missing = [...wanted].filter((slug) => !matched.has(slug)); + const rows = Array.isArray(props["airports"]) ? props["airports"] : []; + const states = Array.isArray(props["states"]) ? props["states"] : []; - if (missing.length > 0) { - console.warn(`parseAirportsJson: no airports found for ${missing.join(", ")} — check the Countries enum.`); - } - } + const airports = rows.filter(isObject).map((row) => new Airport({ + "name": toStringField(row["name"]), + "icao": toStringField(row["icao"]), + "iata": toStringField(row["iata"]), + "lat": null, + "lon": null, + "alt": null, + "country": countryName, + })); + + const statePaths = states + .filter((state) => isObject(state) && typeof state["url"] === "string" && state["url"] !== "") + .map((state) => state["url"]); - return airports; + return { airports, statePaths }; } -module.exports = { parseAirlinesHtml, parseAirportsJson, countryToSlug }; +module.exports = { parseAirlinesHtml, parseInertiaPage, parseAirportsIndex, parseAirportsPage, countryToSlug }; diff --git a/nodejs/package.json b/nodejs/package.json index e6efbcf..2a5f367 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -6,7 +6,7 @@ "types": "./FlightRadarAPI/index.d.ts", "scripts": { "test": "mocha tests --timeout 10000", - "test:offline": "mocha tests/testParsersOffline.js tests/testRequestPolicy.js tests/testRequestTransport.js tests/testFeedRetry.js --timeout 10000", + "test:offline": "mocha tests/testParsersOffline.js tests/testAirportsOffline.js tests/testRequestPolicy.js tests/testRequestTransport.js tests/testFeedRetry.js --timeout 10000", "test:integration": "mocha tests/testApi.js tests/testSnapshots.js --timeout 10000", "test:types": "tsd", "lint": "eslint ." diff --git a/nodejs/tests/fixtures/airlines.html b/nodejs/tests/fixtures/airlines.html index 3c56b99..020dfd5 100644 --- a/nodejs/tests/fixtures/airlines.html +++ b/nodejs/tests/fixtures/airlines.html @@ -1,60 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- LATAM Airlines - xxLA / LAN340 aircraft
- Gol - xxG3 / GLO140 aircraft
- Delta Air Lines - xxDL / DAL900 aircraft
- Sky2 - xxSK10 aircraft
- SkyTeam - xxSKT15 aircraft
placeholder
- Should Skip -
- + + +Airlines - Flightradar24 + +
+
+

Largest fleets

+ All +
+ +
+
+

All airlines

+ +
+ + diff --git a/nodejs/tests/fixtures/airports.json b/nodejs/tests/fixtures/airports.json deleted file mode 100644 index 924f9db..0000000 --- a/nodejs/tests/fixtures/airports.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1786010717", - "rows": [ - { - "name": "Sao Paulo Guarulhos International Airport", - "iata": "GRU", - "icao": "SBGR", - "lat": -23.429991, - "lon": -46.4674, - "country": "Brazil", - "alt": 2436 - }, - { - "name": "Rio de Janeiro Galeao International Airport", - "iata": "GIG", - "icao": "SBGL", - "lat": -22.805696, - "lon": -43.25523, - "country": "Brazil", - "alt": 22 - }, - { - "name": "Sao Paulo Congonhas Airport", - "iata": "CGH", - "icao": "SBSP", - "lat": -23.627348, - "lon": -46.655987, - "country": "Brazil", - "alt": 2614 - }, - { - "name": "Los Angeles International Airport", - "iata": "LAX", - "icao": "KLAX", - "lat": 33.94252, - "lon": -118.406998, - "country": "United States", - "alt": 125 - }, - { - "name": "New York John F. Kennedy International Airport", - "iata": "JFK", - "icao": "KJFK", - "lat": 40.639751, - "lon": -73.7789, - "country": "United States", - "alt": 14 - }, - { - "name": "A Coruna Airport", - "iata": "LCG", - "icao": "LECO", - "lat": 43.302059, - "lon": -8.37725, - "country": "Spain", - "alt": 326 - }, - { - "name": "Ann Airport", - "iata": "VBA", - "icao": "VYAN", - "lat": 19.770643, - "lon": 94.026405, - "country": "Myanmar (Burma)", - "alt": "43" - }, - { - "name": "Broken Coords Airport", - "iata": "BAD", - "icao": "SBAD", - "lat": "n/a", - "lon": "", - "country": "Brazil", - "alt": 0 - } - ] -} diff --git a/nodejs/tests/fixtures/airports_country.html b/nodejs/tests/fixtures/airports_country.html new file mode 100644 index 0000000..833d4f4 --- /dev/null +++ b/nodejs/tests/fixtures/airports_country.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/nodejs/tests/fixtures/airports_country.json b/nodejs/tests/fixtures/airports_country.json new file mode 100644 index 0000000..09b4811 --- /dev/null +++ b/nodejs/tests/fixtures/airports_country.json @@ -0,0 +1,51 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "uruguay" + }, + "query": [] + }, + "country": { + "id": 238, + "name": "Uruguay", + "slug": "uruguay" + }, + "airports": [ + { + "id": 787, + "name": "Colonia Laguna de Los Patos International Airport", + "city": "Colonia", + "iata": "CYR", + "icao": "SUCA", + "total": 8 + }, + { + "id": 2349, + "name": "Montevideo Carrasco International Airport", + "city": "Montevideo", + "iata": "MVD", + "icao": "SUMU", + "total": 632 + } + ], + "airportImages": { + "SUMU": { + "thumbnail": "https://cdn.jetphotos.com/200/5/1.jpg", + "copyright": "Someone" + } + }, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/uruguay", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/fixtures/airports_country_states.json b/nodejs/tests/fixtures/airports_country_states.json new file mode 100644 index 0000000..ff8bca1 --- /dev/null +++ b/nodejs/tests/fixtures/airports_country_states.json @@ -0,0 +1,41 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "brazil" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "states": [ + { + "code": "RJ", + "name": "Rio de Janeiro", + "total": 1, + "url": "/data/airports/brazil/rj" + }, + { + "code": "SP", + "name": "Sao Paulo", + "total": 2, + "url": "/data/airports/brazil/sp" + } + ], + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/fixtures/airports_index.html b/nodejs/tests/fixtures/airports_index.html new file mode 100644 index 0000000..a55ed2f --- /dev/null +++ b/nodejs/tests/fixtures/airports_index.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/nodejs/tests/fixtures/airports_state_rj.json b/nodejs/tests/fixtures/airports_state_rj.json new file mode 100644 index 0000000..77d7a6b --- /dev/null +++ b/nodejs/tests/fixtures/airports_state_rj.json @@ -0,0 +1,44 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "rj" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "RJ", + "name": "Rio de Janeiro", + "slug": "rj" + }, + "airports": [ + { + "id": 1252, + "name": "Rio de Janeiro Galeao International Airport", + "city": "Rio de Janeiro", + "iata": "GIG", + "icao": "SBGL", + "total": 2500 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/rj", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/fixtures/airports_state_sp.json b/nodejs/tests/fixtures/airports_state_sp.json new file mode 100644 index 0000000..46c60c6 --- /dev/null +++ b/nodejs/tests/fixtures/airports_state_sp.json @@ -0,0 +1,52 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "sp" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "SP", + "name": "Sao Paulo", + "slug": "sp" + }, + "airports": [ + { + "id": 1255, + "name": "Sao Paulo Guarulhos International Airport", + "city": "Sao Paulo", + "iata": "GRU", + "icao": "SBGR", + "total": 6796 + }, + { + "id": 1254, + "name": "Sao Paulo Congonhas Airport", + "city": "Sao Paulo", + "iata": "CGH", + "icao": "SBSP", + "total": 3100 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/sp", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/testAirportsOffline.js b/nodejs/tests/testAirportsOffline.js new file mode 100644 index 0000000..27f2f0d --- /dev/null +++ b/nodejs/tests/testAirportsOffline.js @@ -0,0 +1,202 @@ +/** + * Offline tests for the page crawl behind getAirports(). + * + * FR24 lists airports one Inertia page per country, and per state for the + * largest countries. These tests pin down which pages the crawl asks for and + * how it reads them, against the bundled fixtures, so the PR gate covers the + * orchestration without reaching FR24. + */ +const fs = require("fs"); +const path = require("path"); +const expect = require("chai").expect; + +const { FlightRadar24API, Countries } = require("../FlightRadarAPI/index"); +const Core = require("../FlightRadarAPI/core"); +const { parseInertiaPage } = require("../FlightRadarAPI/parsers"); + +const FIXTURES = path.join(__dirname, "fixtures"); +const load = (name) => fs.readFileSync(path.join(FIXTURES, name), "utf-8"); + +const INDEX_VERSION = "919b94d2a33f365a34f99828f6eeec6a"; + +// Every page the fixtures model, by the path FR24 serves it at. +const PAGES = { + "/data/airports": "airports_index.html", + "/data/airports/uruguay": "airports_country.json", + "/data/airports/brazil": "airports_country_states.json", + "/data/airports/brazil/sp": "airports_state_sp.json", + "/data/airports/brazil/rj": "airports_state_rj.json", + "/data/airports/myanmar-burma": "airports_country.html", +}; + +/** + * The Inertia page object of a fixture, as the request layer would decode a JSON body. + * + * @param {string} fixture + * @return {object} + */ +function pageOf(fixture) { + if (fixture.endsWith(".json")) return JSON.parse(load(fixture)); + + const page = parseInertiaPage(load(fixture)); + expect(page).to.not.equal(null); + return page; +} + +/** + * Swap in a client double that serves the fixture pages the way FR24 does. + * + * A request carrying the inertia headers gets the page JSON when the version + * matches and a 409 otherwise; a plain request gets the HTML. The index page + * only exists as HTML, like the real one as the crawl fetches it. + * + * @param {string|null} [version] - Asset version the fake deployment runs + * @return {{api: FlightRadar24API, calls: Array, paths: Function}} + */ +function apiWithFixturePages(version = INDEX_VERSION) { + const api = new FlightRadar24API(); + const calls = []; + + const serve = async (url, options = {}) => { + calls.push({ url, options }); + const pagePath = url.replace(Core.flightRadarBaseUrl, ""); + let fixture = PAGES[pagePath] ?? "airports_index.html"; + const headers = options.headers ?? {}; + + if (headers["x-inertia"] === "true") { + if (headers["x-inertia-version"] !== version) { + expect(options.allowedErrorCodes ?? [], "a 409 must not raise").to.include(409); + return { content: "", statusCode: 409, cookies: {} }; + } + return { content: pageOf(fixture), statusCode: 200, cookies: {} }; + } + + if (fixture.endsWith(".json")) { + fixture = fixture.replace(".json", ".html"); + + if (!fs.existsSync(path.join(FIXTURES, fixture))) fixture = "airports_country.html"; + } + return { content: load(fixture), statusCode: 200, cookies: {} }; + }; + + api.__client = { request: serve, requestStandalone: serve }; + + const paths = () => calls.map((call) => call.url.replace(Core.flightRadarBaseUrl, "")); + + return { api, calls, paths }; +} + +/** + * Run fn while capturing console.warn, returning what it printed. + * + * @param {Function} fn + * @return {Promise>} + */ +async function capturingWarnings(fn) { + const warnings = []; + const original = console.warn; + console.warn = (...args) => warnings.push(args.join(" ")); + + try { + await fn(); + } + finally { + console.warn = original; + } + return warnings; +} + + +describe("getAirports() page crawl (offline)", function() { + it("crawls the country, then its states", async function() { + const { api, paths } = apiWithFixturePages(); + + const airports = await api.getAirports([Countries.BRAZIL]); + + expect(airports.map((a) => a.iata)).to.deep.equal(["GIG", "GRU", "CGH"]); + expect(new Set(airports.map((a) => a.country))).to.deep.equal(new Set(["Brazil"])); + expect(paths()).to.deep.equal([ + "/data/airports", "/data/airports/brazil", "/data/airports/brazil/rj", "/data/airports/brazil/sp", + ]); + }); + + it("asks for the page JSON with the index version", async function() { + const { api, calls } = apiWithFixturePages(); + + await api.getAirports(Countries.URUGUAY); + + const [index, country] = calls; + expect(index.options.headers).to.not.have.property("x-inertia"); + expect(country.options.headers["x-inertia"]).to.equal("true"); + expect(country.options.headers["x-inertia-version"]).to.equal(INDEX_VERSION); + }); + + it("falls back to the HTML when the assets were redeployed", async function() { + const { api, calls, paths } = apiWithFixturePages("something-newer"); + + const airports = await api.getAirports(["uruguay"]); + + // The index answered with a stale version, the JSON request got a 409 + // and the HTML page was read instead. + expect(airports).to.have.lengthOf(1); + expect(paths()).to.deep.equal(["/data/airports", "/data/airports/uruguay", "/data/airports/uruguay"]); + expect(calls[2].options.headers).to.deep.equal(Core.htmlHeaders); + }); + + it("walks the whole index when no country is given", async function() { + const { api, paths } = apiWithFixturePages(); + + const airports = await api.getAirports(); + + expect(airports.map((a) => a.iata)).to.deep.equal(["GIG", "GRU", "CGH", "RGN", "CYR", "MVD"]); + expect(paths()).to.deep.equal([ + "/data/airports", + "/data/airports/brazil", "/data/airports/myanmar-burma", "/data/airports/uruguay", + "/data/airports/brazil/rj", "/data/airports/brazil/sp", + ]); + }); + + it("accepts mixed spellings and drops repeats", async function() { + const { api, paths } = apiWithFixturePages(); + + const airports = await api.getAirports(["Uruguay", Countries.URUGUAY, "uruguay"]); + + expect(airports.map((a) => a.iata)).to.deep.equal(["CYR", "MVD"]); + expect(paths()).to.deep.equal(["/data/airports", "/data/airports/uruguay"]); + }); + + it("makes no request for an empty filter", async function() { + const { api, calls } = apiWithFixturePages(); + + expect(await api.getAirports([])).to.deep.equal([]); + expect(await api.getAirports(new Set())).to.deep.equal([]); + expect(calls).to.deep.equal([]); + }); + + it("rejects a non-iterable filter", async function() { + const { api, calls } = apiWithFixturePages(); + + let error = null; + try { + await api.getAirports(42); + } + catch (caught) { + error = caught; + } + + expect(error).to.be.an.instanceOf(TypeError); + expect(calls).to.deep.equal([]); + }); + + it("returns nothing and warns for an unknown country", async function() { + const { api } = apiWithFixturePages(); + let airports; + + const warnings = await capturingWarnings(async () => { + airports = await api.getAirports(["atlantis"]); + }); + + expect(airports).to.deep.equal([]); + expect(warnings.join("\n")).to.include("no airports found for atlantis"); + }); +}); diff --git a/nodejs/tests/testApi.js b/nodejs/tests/testApi.js index c20ddaa..caa7db9 100644 --- a/nodejs/tests/testApi.js +++ b/nodejs/tests/testApi.js @@ -44,6 +44,10 @@ describe("Testing FlightRadarAPI version " + version, function() { }); describe("Getting Airports", function() { + // The listing is one page per country and state, crawled maxWorkers at + // a time: the whole index takes well over mocha's default 10 seconds. + this.timeout(60000); + const expected = 1800; const countries = [Countries.BRAZIL, Countries.UNITED_STATES]; diff --git a/nodejs/tests/testParsersOffline.js b/nodejs/tests/testParsersOffline.js index 0326870..4e3eb5d 100644 --- a/nodejs/tests/testParsersOffline.js +++ b/nodejs/tests/testParsersOffline.js @@ -11,29 +11,22 @@ const fs = require("fs"); const path = require("path"); const expect = require("chai").expect; -const { parseAirlinesHtml, parseAirportsJson, countryToSlug } = require("../FlightRadarAPI/parsers"); +const { + parseAirlinesHtml, parseInertiaPage, parseAirportsIndex, parseAirportsPage, countryToSlug, +} = require("../FlightRadarAPI/parsers"); const { Countries } = require("../FlightRadarAPI/core"); const FIXTURES = path.join(__dirname, "fixtures"); const load = (name) => fs.readFileSync(path.join(FIXTURES, name), "utf-8"); - -// Keep in step with the same list in python/tests/test_parsers_offline.py. -const NUMERIC_CASES = [ - ["whitespace", " ", null], - ["an array holding a number", [43], null], - ["a coordinate with a hemisphere suffix", "43.30 N", null], - ["an exponent overflowing a double", "1e999", null], - ["400 plain digits", "1".repeat(400), null], - ["a whole number as a string", "2436", 2436], - ["a negative decimal", "-23.4", -23.4], - ["a genuine zero", 0, 0], -]; +const loadJson = (name) => JSON.parse(load(name)); describe("parseAirlinesHtml (offline)", function() { let airlines; + let byName; before(function() { airlines = parseAirlinesHtml(load("airlines.html")); + byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); }); it("extracts known airline rows", function() { @@ -44,7 +37,6 @@ describe("parseAirlinesHtml (offline)", function() { }); it("splits IATA and ICAO when both are present", function() { - const byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); expect(byName["LATAM Airlines"].IATA).to.equal("LA"); expect(byName["LATAM Airlines"].ICAO).to.equal("LAN"); expect(byName["Gol"].IATA).to.equal("G3"); @@ -52,182 +44,167 @@ describe("parseAirlinesHtml (offline)", function() { }); it("handles IATA-only or ICAO-only rows", function() { - const byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); expect(byName["Sky2"].IATA).to.equal("SK"); expect(byName["Sky2"].ICAO).to.equal(null); expect(byName["SkyTeam"].ICAO).to.equal("SKT"); expect(byName["SkyTeam"].IATA).to.equal(null); }); - it("parses aircraft count from the last column", function() { - const byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); + it("parses the aircraft count", function() { expect(byName["LATAM Airlines"].n_aircrafts).to.equal(340); expect(byName["Gol"].n_aircrafts).to.equal(140); + // Thousands separator, as the page prints large fleets. + expect(byName["Delta Air Lines"].n_aircrafts).to.equal(1900); + // Only the "Largest fleets" section carries a count. + expect(byName["Sky2"].n_aircrafts).to.equal(null); + }); + + it("merges an airline listed in two sections", function() { + // Delta and Gol appear under "Largest fleets" and again under "All airlines". + const names = airlines.map((a) => a.Name); + expect(names.filter((name) => name === "Delta Air Lines")).to.have.lengthOf(1); + expect(names.filter((name) => name === "Gol")).to.have.lengthOf(1); }); - it("skips rows without notranslate td or with wrong href", function() { - expect(airlines.length).to.equal(5); + it("keeps a code-shaped name", function() { + expect(byName["TAP"]).to.deep.equal({ "Name": "TAP", "ICAO": "TAP", "IATA": "TP", "n_aircrafts": null }); + }); + + it("skips invalid rows", function() { + // 6 airlines; the section link, the entry with no link, the wrong href + // and the one-letter name must be skipped. + expect(airlines).to.have.lengthOf(6); + expect(airlines.map((a) => a.Name)).to.not.include("All"); }); it("returns an empty list for empty or unrecognised HTML", function() { expect(parseAirlinesHtml("")).to.deep.equal([]); - expect(parseAirlinesHtml("

no tbody here

")).to.deep.equal([]); + expect(parseAirlinesHtml("

no airlines here

")).to.deep.equal([]); }); }); -describe("parseAirportsJson (offline)", function() { - let airports; - before(function() { - airports = parseAirportsJson(load("airports.json")); +describe("parseInertiaPage (offline)", function() { + it("reads the data-page attribute", function() { + const page = parseInertiaPage(load("airports_index.html")); + expect(page.component).to.equal("Data/Airports"); + expect(page.version).to.equal("919b94d2a33f365a34f99828f6eeec6a"); + expect(page.props).to.have.property("airportsByCountry"); }); - it("extracts iata, icao, name, country and position", function() { - const byIata = Object.fromEntries(airports.map((a) => [a.iata, a])); - expect(byIata["GRU"]).to.exist; - expect(byIata["GIG"]).to.exist; - expect(byIata["GRU"].icao).to.equal("SBGR"); - expect(byIata["GRU"].country).to.equal("Brazil"); - expect(byIata["GRU"].latitude).to.be.closeTo(-23.429991, 1e-6); - expect(byIata["GRU"].longitude).to.be.closeTo(-46.4674, 1e-6); - expect(byIata["GRU"].altitude).to.equal(2436); + it("accepts a decoded JSON body", function() { + const body = loadJson("airports_country.json"); + expect(parseInertiaPage(body)).to.equal(body); }); - it("keeps every country when no filter is given", function() { - const countries = new Set(airports.map((a) => a.country)); - expect(countries).to.include("Brazil"); - expect(countries).to.include("United States"); - expect(countries).to.include("Spain"); - }); + it("reads a body that arrived as raw bytes", function() { + // request() returns an ArrayBuffer on an unexpected content-type. + const html = load("airports_index.html"); - it("filters by the slugs from the Countries enum", function() { - const filtered = parseAirportsJson(load("airports.json"), ["united-states"]); - expect(filtered.length).to.be.above(0); - for (const airport of filtered) { - expect(airport.country).to.equal("United States"); + for (const payload of [Buffer.from(html), new TextEncoder().encode(html), new TextEncoder().encode(html).buffer]) { + expect(parseInertiaPage(payload).version, payload.constructor.name).to.equal("919b94d2a33f365a34f99828f6eeec6a"); } }); - it("accepts several countries at once", function() { - const filtered = parseAirportsJson(load("airports.json"), ["brazil", "spain"]); - const countries = new Set(filtered.map((a) => a.country)); - expect([...countries].sort()).to.deep.equal(["Brazil", "Spain"]); - }); - - it("invalid coordinates become null (regression test for 0,0 fallback)", function() { - const bad = airports.find((a) => a.iata === "BAD"); - expect(bad.latitude).to.equal(null); - expect(bad.longitude).to.equal(null); + it("rejects unrecognised payloads", function() { + expect(parseInertiaPage("")).to.equal(null); + expect(parseInertiaPage("
")).to.equal(null); + expect(parseInertiaPage("
")).to.equal(null); + expect(parseInertiaPage({ component: "Data/Airports" })).to.equal(null); }); +}); - it("keeps FR24's own country spelling and still slugifies it for URLs", function() { - const ann = airports.find((airport) => airport.iata === "VBA"); - expect(ann.country).to.equal("Myanmar (Burma)"); - expect(countryToSlug(ann.country)).to.equal("myanmar-burma"); - expect(ann.altitude).to.equal(43); +describe("parseAirportsIndex (offline)", function() { + it("lists every country with its slug", function() { + const countries = parseAirportsIndex(load("airports_index.html")); + expect(countries.map((c) => c.slug)).to.deep.equal(["brazil", "myanmar-burma", "uruguay"]); + expect(countries[1].name).to.equal("MYANMAR (BURMA)"); }); - it("matches a filter written with either spelling of the country", function() { - for (const slug of ["myanmar-burma", "Myanmar (Burma)"]) { - expect(parseAirportsJson(load("airports.json"), [slug]).length).to.equal(1); + it("produces slugs that match the Countries enum", function() { + const known = new Set(Object.values(Countries)); + + for (const country of parseAirportsIndex(load("airports_index.html"))) { + expect(known.has(country.slug), country.slug).to.equal(true); } }); - it("rejects numeric-looking junk instead of truncating it", function() { - const payload = JSON.stringify({ rows: [{ - name: "Junk Airport", iata: "JNK", icao: "JJNK", - lat: "43.30 N", lon: -8.37725, country: "Spain", alt: "-1", - }] }); - const [airport] = parseAirportsJson(payload); - - // One bad coordinate drops both; altitude is independent and survives. - expect(airport.latitude).to.equal(null); - expect(airport.longitude).to.equal(null); - expect(airport.altitude).to.equal(-1); + it("returns an empty list for an invalid payload", function() { + expect(parseAirportsIndex("")).to.deep.equal([]); + expect(parseAirportsIndex({ props: {} })).to.deep.equal([]); + expect(parseAirportsIndex({ props: { airportsByCountry: [1, "x", { name: null }] } })).to.deep.equal([]); }); +}); - it("agrees with the Python port on Unicode digits and stray control bytes", function() { - // Each port used to accept what the other rejected. - const read = (value) => { - const payload = JSON.stringify({ rows: [{ - name: "X", iata: "XXX", icao: "XXXX", country: "Spain", lat: value, lon: value, alt: value, - }] }); - return parseAirportsJson(payload)[0].altitude; - }; - expect(read("\u0664\u0663"), "Arabic-Indic digits").to.equal(null); - expect(read("\ufeff43"), "leading U+FEFF").to.equal(null); - expect(read("\u001c43"), "leading U+001C").to.equal(null); - expect(read(" 43 "), "plain spaces still trimmed").to.equal(43); - }); +describe("parseAirportsPage (offline)", function() { + it("extracts the airports of a country", function() { + const { airports, statePaths } = parseAirportsPage(loadJson("airports_country.json")); + expect(statePaths).to.deep.equal([]); + expect(airports.map((a) => a.iata)).to.deep.equal(["CYR", "MVD"]); - it("returns an empty list for an unknown country", function() { - expect(parseAirportsJson(load("airports.json"), ["atlantis"])).to.deep.equal([]); + const mvd = airports[1]; + expect(mvd.name).to.equal("Montevideo Carrasco International Airport"); + expect(mvd.icao).to.equal("SUMU"); + expect(mvd.country).to.equal("Uruguay"); }); - it("survives numbers written raw in the JSON that no double can hold", function() { - // Written as text: a JS literal is already rounded before it reaches here. - for (const literal of ["1".repeat(400), "9007199254740993", "-1".concat("1".repeat(399))]) { - const payload = `{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":${literal},"lon":1,"alt":2}]}`; - const [airport] = parseAirportsJson(payload); + it("carries no position: the listing has none", function() { + const { airports } = parseAirportsPage(loadJson("airports_country.json")); - expect(airport.latitude, literal.slice(0, 12)).to.satisfy( - (latitude) => latitude === null || Number.isFinite(latitude), - ); + for (const airport of airports) { + expect(airport.latitude).to.equal(null); + expect(airport.longitude).to.equal(null); + expect(airport.altitude).to.equal(null); } }); - it("keeps text fields as strings whatever the feed sends", function() { - // Anything else breaks getCountryFlag(airport.country). - for (const literal of ["null", "0", "false", "true", "[]", "{}", "123"]) { - const payload = `{"rows":[{"name":${literal},"iata":${literal},"icao":${literal},` + - `"country":${literal},"lat":1,"lon":2,"alt":3}]}`; - const [airport] = parseAirportsJson(payload); - - expect([airport.name, airport.iata, airport.icao, airport.country], literal) - .to.deep.equal(["", "", "", ""]); - } + it("returns the state pages of a split country", function() { + const { airports, statePaths } = parseAirportsPage(loadJson("airports_country_states.json")); + expect(airports).to.deep.equal([]); + expect(statePaths).to.deep.equal(["/data/airports/brazil/rj", "/data/airports/brazil/sp"]); }); - it("parses a body that arrived as raw bytes", function() { - // request() returns an ArrayBuffer on an unexpected content-type. - const json = load("airports.json"); + it("keeps the country name on a state page", function() { + const { airports, statePaths } = parseAirportsPage(loadJson("airports_state_sp.json")); + expect(statePaths).to.deep.equal([]); + expect(airports.map((a) => a.iata)).to.deep.equal(["GRU", "CGH"]); + expect(new Set(airports.map((a) => a.country))).to.deep.equal(new Set(["Brazil"])); + }); - for (const payload of [Buffer.from(json), new TextEncoder().encode(json), new TextEncoder().encode(json).buffer]) { - expect(parseAirportsJson(payload).length, payload.constructor.name).to.equal(airports.length); - } + it("reads an HTML page", function() { + const { airports } = parseAirportsPage(load("airports_country.html")); + expect(airports.map((a) => a.iata)).to.deep.equal(["RGN"]); + // FR24 spells it "Myanmar (Burma)" on the page and "myanmar-burma" in URLs. + expect(airports[0].country).to.equal("Myanmar (Burma)"); + expect(countryToSlug(airports[0].country)).to.equal(Countries.MYANMAR_BURMA); }); it("skips rows that are not objects, as the Python port does", function() { - const payload = "{\"rows\":[[1,2,3],\"x\",7,null," + - "{\"name\":\"Real\",\"iata\":\"RRR\",\"icao\":\"RRRR\",\"country\":\"Spain\",\"lat\":1,\"lon\":2,\"alt\":3}]}"; - - expect(parseAirportsJson(payload).map((airport) => airport.iata)).to.deep.equal(["RRR"]); + const payload = { props: { country: { name: "Nowhere" }, airports: [1, "x", null, [1, 2], { iata: "RRR" }] } }; + const { airports } = parseAirportsPage(payload); + expect(airports.map((a) => a.iata)).to.deep.equal(["RRR"]); }); - it("returns an empty list when the feed has no rows", function() { - expect(parseAirportsJson("")).to.deep.equal([]); - expect(parseAirportsJson("{}")).to.deep.equal([]); - expect(parseAirportsJson("not json")).to.deep.equal([]); + it("keeps text fields as strings whatever the page sends", function() { + // Anything else breaks getCountryFlag(airport.country). + const payload = { props: { country: { name: ["Brazil"] }, airports: [{ name: 42, iata: null, icao: { x: 1 } }] } }; + const [airport] = parseAirportsPage(payload).airports; + expect([airport.name, airport.iata, airport.icao, airport.country]).to.deep.equal(["", "", "", ""]); }); -}); + it("ignores malformed states", function() { + const payload = { props: { states: [{ url: "/data/airports/x/a" }, { url: "" }, { code: "B" }, "junk", { url: 3 }] } }; + expect(parseAirportsPage(payload)).to.deep.equal({ airports: [], statePaths: ["/data/airports/x/a"] }); + }); -describe("parseAirportsJson numeric coercion", function() { - for (const [label, input, expected] of NUMERIC_CASES) { - it(`reads ${label} as ${JSON.stringify(expected)}`, function() { - const payload = JSON.stringify({ rows: [{ - name: "X", iata: "XXX", icao: "XXXX", country: "Spain", - lat: input, lon: input, alt: input, - }] }); - const [airport] = parseAirportsJson(payload); - - expect(airport.latitude, "latitude").to.equal(expected); - expect(airport.longitude, "longitude").to.equal(expected); - expect(airport.altitude, "altitude").to.equal(expected); - }); - } + it("returns nothing for an invalid payload", function() { + const empty = { airports: [], statePaths: [] }; + expect(parseAirportsPage("")).to.deep.equal(empty); + expect(parseAirportsPage("not a data page")).to.deep.equal(empty); + expect(parseAirportsPage({ props: {} })).to.deep.equal(empty); + }); }); diff --git a/python/FlightRadarAPI/api.py b/python/FlightRadarAPI/api.py index e440e8a..d2d6646 100644 --- a/python/FlightRadarAPI/api.py +++ b/python/FlightRadarAPI/api.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import dataclasses +import logging import math from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict, Iterable, List, Optional, Tuple, Union @@ -11,9 +12,11 @@ from .entities.flight import Flight from .errors import AirportNotFoundError, LoginError from .flight_tracker_config import FlightTrackerConfig -from .parsers import country_to_slug, parse_airlines_html, parse_airports_json +from .parsers import country_to_slug, parse_airlines_html, parse_airports_index, parse_airports_page, parse_inertia_page from .request import APIClient, RetryPolicy +_logger = logging.getLogger(__name__) + # Some FR24 live-feed backends answer 200 with a well-formed envelope but no # flight entries -- indistinguishable from a legitimately empty result. The # "AWSALB" cookie then pins the session to that backend, so dropping it is what @@ -193,6 +196,12 @@ def get_airports( """ Return a list with all airports, optionally narrowed to some countries. + FR24 lists airports one page per country, and one per state for the + largest countries, so this fetches one page per country asked for (a few + hundred for every airport), `max_workers` at a time. The listing carries + the name, codes and country only: use get_airport(code) for the position + and altitude of an airport. + :param countries: Country names from the Countries enum, or their slug strings, as any iterable or a single value. Every country when omitted. """ @@ -207,13 +216,82 @@ def get_airports( if wanted is not None and len(wanted) == 0: return [] - response = self.__client.request( - Core.airports_json_url, headers=Core.json_headers, timeout=self.timeout, + # The index names every country, and declares the asset version the + # JSON requests below have to quote. + response = self.__client.request(Core.airports_data_url, headers=Core.html_headers, timeout=self.timeout) + index = parse_inertia_page(response.get_bytes_content()) + version = index.get("version") if index is not None else None + + if wanted is None: + slugs = [country["slug"] for country in parse_airports_index(index or {})] + else: + slugs = [country_to_slug(country) for country in wanted] + + slugs = list(dict.fromkeys(slug for slug in slugs if slug)) + version = version if isinstance(version, str) else None + + country_pages = self.__get_airport_pages( + [Core.airports_by_country_url.format(slug) for slug in slugs], version, ) - # get_content(), not get_json_content(): an html body reaches the parser's - # guard; a malformed JSON body still raises. The parser slugifies `wanted`. - return parse_airports_json(response.get_content(), wanted) + # Second pass for the countries split by state, every state at once. + state_owners = [index for index, (_, paths) in enumerate(country_pages) for _ in paths] + state_pages = self.__get_airport_pages( + [ + path if path.startswith("http") else Core.flightradar_base_url + path + for _, paths in country_pages for path in paths + ], + version, + ) + + # Grouped by country, in the order asked for, each country's states after it. + airports_by_country: List[List[Airport]] = [list(found) for found, _ in country_pages] + + for owner, (found, _) in zip(state_owners, state_pages): + airports_by_country[owner].extend(found) + + airports = [airport for found in airports_by_country for airport in found] + + # A country FR24 does not know answers with the index page, which lists + # neither airports nor states. + missing = [slug for slug, (found, paths) in zip(slugs, country_pages) if not found and not paths] + + if missing: + _logger.warning( + "get_airports: no airports found for %s — check the Countries enum.", ", ".join(missing), + ) + + return airports + + def __get_airport_pages(self, urls: List[str], version: Optional[str]) -> List[Tuple[List[Airport], List[str]]]: + """ + Fetch and parse the airport listing pages at `urls`, in order, max_workers at a time. + """ + if not urls: + return [] + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + return list(executor.map(lambda url: parse_airports_page(self.__get_inertia_page(url, version)), urls)) + + def __get_inertia_page(self, url: str, version: Optional[str]) -> Union[Dict, bytes]: + """ + Fetch a FR24 data page as its Inertia page object, falling back to the HTML. + + With the asset version FR24 answers the bare page JSON, a fraction of the + HTML. A 409 means the assets were redeployed since the version was read: + the HTML carries the same page object and needs no version. + """ + if version is not None: + headers = {**Core.inertia_headers, "x-inertia-version": version} + response = self.__client.request_standalone( + url, headers=headers, timeout=self.timeout, allowed_error_codes=[409], + ) + + if response.get_status_code() != 409: + return response.get_content() + + response = self.__client.request_standalone(url, headers=Core.html_headers, timeout=self.timeout) + return response.get_bytes_content() def get_bookmarks(self) -> Dict: """ diff --git a/python/FlightRadarAPI/core.py b/python/FlightRadarAPI/core.py index 5948e6c..93fcd5c 100644 --- a/python/FlightRadarAPI/core.py +++ b/python/FlightRadarAPI/core.py @@ -30,10 +30,12 @@ class Core: # Airports data URLs. api_airport_data_url = api_flightradar_base_url + "/airport.json" airport_data_url = flightradar_base_url + "/airports/traffic-stats/?airport={}" + # The listing is split by country, and by state for the largest countries. + # These are Inertia pages: the data rides JSON-encoded in the data-page + # attribute of the HTML, or comes as the whole body when the request carries + # `inertia_headers` and the asset version the index page declares. airports_data_url = flightradar_base_url + "/data/airports" - - # Every airport in one payload; the per-country pages above are client-rendered. - airports_json_url = flightradar_base_url + "/_json/airports.php" + airports_by_country_url = airports_data_url + "/{}" # Airlines data URL. airlines_data_url = flightradar_base_url + "/data/airlines" @@ -86,6 +88,12 @@ class Core: image_headers = headers.copy() image_headers["accept"] = "image/gif, image/jpg, image/jpeg, image/png" + # Asks an Inertia page for its page object instead of the HTML around it. + # `x-inertia-version` must match the deployed assets or FR24 answers 409. + inertia_headers = json_headers.copy() + inertia_headers["x-inertia"] = "true" + inertia_headers["x-requested-with"] = "XMLHttpRequest" + html_headers = { "accept": ( "text/html,application/xhtml+xml,application/xml;q=0.9," diff --git a/python/FlightRadarAPI/parsers.py b/python/FlightRadarAPI/parsers.py index 77667ad..59792b2 100644 --- a/python/FlightRadarAPI/parsers.py +++ b/python/FlightRadarAPI/parsers.py @@ -2,86 +2,111 @@ import json import logging -import math import re import unicodedata from enum import Enum -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from bs4 import BeautifulSoup +from bs4.element import Tag from .entities.airport import Airport _logger = logging.getLogger(__name__) -# ASCII only: \d would otherwise match Unicode digits such as "٤٣". -NUMERIC_PATTERN = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$", re.ASCII) +# An airline code badge: a two-letter IATA code or a three-letter ICAO code. +# ASCII only, so a stray Unicode letter does not pass for a code. +AIRLINE_CODE_PATTERN = re.compile(r"^[A-Z0-9]{2,3}$", re.ASCII) -# str.strip() and trim() disagree on U+001C-U+001F and U+FEFF: pick one set. -SURROUNDING_SPACE = " \t\n\r\f\v" +# A fleet size as the page prints it: "340" or "3,878". +AIRCRAFT_COUNT_PATTERN = re.compile(r"^\d{1,3}(?:[,.]\d{3})*$|^\d+$", re.ASCII) -# Number.MAX_SAFE_INTEGER, the largest integer JavaScript holds exactly. -MAX_EXACT_INTEGER = 2 ** 53 - 1 +AIRLINES_PATH_PREFIX = "/data/airlines/" + + +def _is_airline_link(href: object) -> bool: + return isinstance(href, str) and href.startswith(AIRLINES_PATH_PREFIX) + + +def _leaf_texts(root: Tag, tag: str) -> List[str]: + """ + Text of the `tag` elements under `root` that hold no other element. + + The name element is left out by its `title` attribute: an airline called + "TAP" would otherwise read as an ICAO code. + """ + return [ + element.get_text(strip=True) + for element in root.find_all(tag) + if element.find() is None and not element.has_attr("title") + ] + + +def _airline_name(link: Tag) -> str: + """ + The airline name of a listing entry. + + The name sits in the element titled with it, with the logo's aria-label as a + fallback; the anchor text also carries the codes and the fleet size. + """ + for attribute in ("title", "aria-label"): + element = link.find(lambda tag, name=attribute: tag.has_attr(name)) + + if isinstance(element, Tag): + return str(element[attribute]).strip() + + return link.get_text(strip=True) def parse_airlines_html(html: bytes) -> List[Dict]: """ - Parse the airlines listing HTML page into a list of airline dicts. + Parse the airlines listing page into a list of airline dicts. + + Every airline is a link to its data page, with the IATA and ICAO codes as + badges next to the name. The "Largest fleets" section repeats a few airlines + with their aircraft count; an airline the page lists twice becomes one entry + carrying everything the page said about it. """ soup = BeautifulSoup(html, "html.parser") - tbody = soup.find("tbody") + links = soup.find_all("a", href=_is_airline_link) - if not tbody: + if not links: _logger.warning( - "parse_airlines_html: no in response — FR24 page layout may have changed." + "parse_airlines_html: no airline links in response — FR24 page layout may have changed." ) return [] - airlines = [] - - for tr in tbody.find_all("tr"): - td_notranslate = tr.find("td", class_="notranslate") + airlines: Dict[str, Dict] = {} - if not td_notranslate: - continue - - a_element = td_notranslate.find("a", href=lambda href: href and href.startswith("/data/airlines")) + for link in links: + name = _airline_name(link) - if not a_element: + if len(name) < 2: continue - airline_name = a_element.get_text(strip=True) + codes = [text for text in _leaf_texts(link, "span") if AIRLINE_CODE_PATTERN.match(text)] + iata = next((code for code in codes if len(code) == 2), None) + icao = next((code for code in codes if len(code) == 3), None) - if len(airline_name) < 2: + # Section links ("All") point under /data/airlines/ too; a code tells + # an airline apart from them. + if iata is None and icao is None: continue - td_elements = tr.find_all("td") - iata = None - icao = None - - if len(td_elements) >= 4: - codes_text = td_elements[3].get_text(strip=True) - - if " / " in codes_text: - parts = codes_text.split(" / ") - if len(parts) == 2: - iata = parts[0].strip() - icao = parts[1].strip() - elif len(codes_text) == 2: - iata = codes_text - elif len(codes_text) == 3: - icao = codes_text - - n_aircrafts = None + n_aircrafts = next( + (int(re.sub(r"\D", "", text)) for text in _leaf_texts(link, "strong") if AIRCRAFT_COUNT_PATTERN.match(text)), + None, + ) - if len(td_elements) >= 5: - aircrafts_text = td_elements[4].get_text(strip=True) - if aircrafts_text: - n_aircrafts = int(aircrafts_text.split(" ", maxsplit=1)[0].strip()) + airline = airlines.setdefault( + str(link["href"]), {"Name": name, "ICAO": None, "IATA": None, "n_aircrafts": None}, + ) - airlines.append({"Name": airline_name, "ICAO": icao, "IATA": iata, "n_aircrafts": n_aircrafts}) + for key, value in (("ICAO", icao), ("IATA", iata), ("n_aircrafts", n_aircrafts)): + if airline[key] is None: + airline[key] = value - return airlines + return list(airlines.values()) def country_to_slug(country: object) -> str: @@ -115,125 +140,103 @@ def _to_text(value: object) -> str: return value if isinstance(value, str) else "" -def _to_number(value: object) -> Optional[Union[int, float]]: +def parse_inertia_page(payload: Union[bytes, str, Dict]) -> Optional[Dict]: """ - Coerce a numeric feed field into a number, or None when it is unusable. - - An unusable coordinate must not become 0.0: that would place the airport in - the Gulf of Guinea instead of marking its position as unknown. + Return the Inertia page object (component, props, version) of a FR24 data page. - Plain whole numbers stay ints, so an altitude reads 2436 in both ports rather - than 2436.0 here -- the feed sends `alt` as an int for most rows and as a - string ("-1") for the rest. Anything written with a decimal point or an - exponent stays a float: "2436.0" and "1e3" are floats here and integral - Numbers in Node, equal in value but not in type. + Served as HTML, the page keeps it JSON-encoded in the data-page attribute of + its root element; asked with Core.inertia_headers, FR24 sends it as the body, + which `request.get_content()` has already decoded into a dict. """ - # bool is an int here, and str([43]) would invent a number. - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - - if isinstance(value, str): - stripped = value.strip(SURROUNDING_SPACE) + if isinstance(payload, dict): + page: object = payload + else: + root = BeautifulSoup(payload, "html.parser").find(lambda tag: tag.has_attr("data-page")) - if not NUMERIC_PATTERN.match(stripped): + if not isinstance(root, Tag): + _logger.warning("parse_inertia_page: no data-page attribute in response — FR24 page layout may have changed.") return None - text: Union[int, float, str] = stripped - else: - text = value - - # One conversion, so overflow is handled once: Python ints have no ceiling. - try: - number = float(text) - except OverflowError: - return None + try: + page = json.loads(str(root["data-page"])) + except ValueError: + _logger.warning("parse_inertia_page: data-page is not valid JSON — FR24 page layout may have changed.") + return None - if not math.isfinite(number): + if not isinstance(page, dict) or not isinstance(page.get("props"), dict): + _logger.warning("parse_inertia_page: no page props in response — FR24 page layout may have changed.") return None - try: - exact = int(text) - except (ValueError, OverflowError): - return number - - # int only when lossless (so -23.4 survives) and only where JavaScript is exact. - return exact if exact == number and abs(exact) <= MAX_EXACT_INTEGER else number + return page -def parse_airports_json( - payload: Union[bytes, str, Dict], countries: Optional[Iterable[object]] = None, -) -> List[Airport]: +def parse_airports_index(payload: Union[bytes, str, Dict]) -> List[Dict]: """ - Parse the airports JSON feed into a list of Airport instances. + Parse the airports index (Core.airports_data_url) into its countries. - :param payload: Body of Core.airports_json_url. - :param countries: Country slugs, or Countries members. Every airport is kept when omitted. + Each entry has the country "name" as FR24 prints it ("UNITED STATES") and the + "slug" of its listing page, which is also its Countries enum value. """ - if isinstance(payload, (bytes, str)): - try: - data = json.loads(payload) - except ValueError: - _logger.warning("parse_airports_json: response is not valid JSON — FR24 feed may have changed.") - return [] - else: - data = payload - - rows = data.get("rows") if isinstance(data, dict) else None + page = parse_inertia_page(payload) + rows = page["props"].get("airportsByCountry") if page is not None else None if not isinstance(rows, list): - _logger.warning('parse_airports_json: no "rows" array in response — FR24 feed may have changed.') + _logger.warning('parse_airports_index: no "airportsByCountry" list in response — FR24 page layout may have changed.') return [] - wanted = {country_to_slug(country) for country in countries} if countries is not None else None - matched = set() - unpositioned = [] - airports = [] + countries = [] for row in rows: if not isinstance(row, dict): continue - # Slugified only when filtering: it is otherwise unused, and this loop - # runs over every airport in the feed. - if wanted is not None: - slug = country_to_slug(row.get("country")) + name = _to_text(row.get("name")) + slug = country_to_slug(name) - if slug not in wanted: - continue - matched.add(slug) + if slug: + countries.append({"name": name, "slug": slug}) - latitude = _to_number(row.get("lat")) - longitude = _to_number(row.get("lon")) + return countries - # One bad coordinate drops both: half a position reads as located. - if latitude is None or longitude is None: - latitude = longitude = None - unpositioned.append(_to_text(row.get("name"))) - airports.append(Airport(basic_info={ - "name": _to_text(row.get("name")), - "icao": _to_text(row.get("icao")), - "iata": _to_text(row.get("iata")), - "lat": latitude, - "lon": longitude, - "alt": _to_number(row.get("alt")), - "country": _to_text(row.get("country")), - })) - - # One line, not one per row: the feed carries every airport. - if unpositioned: - _logger.warning( - "parse_airports_json: %d airport(s) had unusable coordinates and carry no position (e.g. %s).", - len(unpositioned), ", ".join(repr(name) for name in unpositioned[:3]), - ) +def parse_airports_page(payload: Union[bytes, str, Dict]) -> Tuple[List[Airport], List[str]]: + """ + Parse a country or state airports page into (airports, state_paths). - if wanted is not None: - missing = sorted(wanted - matched) + A country with many airports lists its states instead, each with the path of + its own page; those paths come back for the caller to fetch. The listing has + no coordinates or altitude: `get_airport(code)` has them for one airport. + """ + page = parse_inertia_page(payload) + + if page is None: + return [], [] + + props = page["props"] + country = props.get("country") + country_name = _to_text(country.get("name")) if isinstance(country, dict) else "" - if missing: - _logger.warning( - "parse_airports_json: no airports found for %s — check the Countries enum.", - ", ".join(missing), - ) + rows = props.get("airports") + states = props.get("states") - return airports + airports = [ + Airport(basic_info={ + "name": _to_text(row.get("name")), + "icao": _to_text(row.get("icao")), + "iata": _to_text(row.get("iata")), + "lat": None, + "lon": None, + "alt": None, + "country": country_name, + }) + for row in (rows if isinstance(rows, list) else []) + if isinstance(row, dict) + ] + + state_paths = [ + state["url"] + for state in (states if isinstance(states, list) else []) + if isinstance(state, dict) and isinstance(state.get("url"), str) and state["url"] + ] + + return airports, state_paths diff --git a/python/tests/fixtures/airlines.html b/python/tests/fixtures/airlines.html index 3c56b99..020dfd5 100644 --- a/python/tests/fixtures/airlines.html +++ b/python/tests/fixtures/airlines.html @@ -1,60 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- LATAM Airlines - xxLA / LAN340 aircraft
- Gol - xxG3 / GLO140 aircraft
- Delta Air Lines - xxDL / DAL900 aircraft
- Sky2 - xxSK10 aircraft
- SkyTeam - xxSKT15 aircraft
placeholder
- Should Skip -
- + + +Airlines - Flightradar24 + +
+
+

Largest fleets

+ All +
+ +
+
+

All airlines

+ +
+ + diff --git a/python/tests/fixtures/airports.json b/python/tests/fixtures/airports.json deleted file mode 100644 index 924f9db..0000000 --- a/python/tests/fixtures/airports.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1786010717", - "rows": [ - { - "name": "Sao Paulo Guarulhos International Airport", - "iata": "GRU", - "icao": "SBGR", - "lat": -23.429991, - "lon": -46.4674, - "country": "Brazil", - "alt": 2436 - }, - { - "name": "Rio de Janeiro Galeao International Airport", - "iata": "GIG", - "icao": "SBGL", - "lat": -22.805696, - "lon": -43.25523, - "country": "Brazil", - "alt": 22 - }, - { - "name": "Sao Paulo Congonhas Airport", - "iata": "CGH", - "icao": "SBSP", - "lat": -23.627348, - "lon": -46.655987, - "country": "Brazil", - "alt": 2614 - }, - { - "name": "Los Angeles International Airport", - "iata": "LAX", - "icao": "KLAX", - "lat": 33.94252, - "lon": -118.406998, - "country": "United States", - "alt": 125 - }, - { - "name": "New York John F. Kennedy International Airport", - "iata": "JFK", - "icao": "KJFK", - "lat": 40.639751, - "lon": -73.7789, - "country": "United States", - "alt": 14 - }, - { - "name": "A Coruna Airport", - "iata": "LCG", - "icao": "LECO", - "lat": 43.302059, - "lon": -8.37725, - "country": "Spain", - "alt": 326 - }, - { - "name": "Ann Airport", - "iata": "VBA", - "icao": "VYAN", - "lat": 19.770643, - "lon": 94.026405, - "country": "Myanmar (Burma)", - "alt": "43" - }, - { - "name": "Broken Coords Airport", - "iata": "BAD", - "icao": "SBAD", - "lat": "n/a", - "lon": "", - "country": "Brazil", - "alt": 0 - } - ] -} diff --git a/python/tests/fixtures/airports_country.html b/python/tests/fixtures/airports_country.html new file mode 100644 index 0000000..833d4f4 --- /dev/null +++ b/python/tests/fixtures/airports_country.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/python/tests/fixtures/airports_country.json b/python/tests/fixtures/airports_country.json new file mode 100644 index 0000000..09b4811 --- /dev/null +++ b/python/tests/fixtures/airports_country.json @@ -0,0 +1,51 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "uruguay" + }, + "query": [] + }, + "country": { + "id": 238, + "name": "Uruguay", + "slug": "uruguay" + }, + "airports": [ + { + "id": 787, + "name": "Colonia Laguna de Los Patos International Airport", + "city": "Colonia", + "iata": "CYR", + "icao": "SUCA", + "total": 8 + }, + { + "id": 2349, + "name": "Montevideo Carrasco International Airport", + "city": "Montevideo", + "iata": "MVD", + "icao": "SUMU", + "total": 632 + } + ], + "airportImages": { + "SUMU": { + "thumbnail": "https://cdn.jetphotos.com/200/5/1.jpg", + "copyright": "Someone" + } + }, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/uruguay", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/fixtures/airports_country_states.json b/python/tests/fixtures/airports_country_states.json new file mode 100644 index 0000000..ff8bca1 --- /dev/null +++ b/python/tests/fixtures/airports_country_states.json @@ -0,0 +1,41 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "brazil" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "states": [ + { + "code": "RJ", + "name": "Rio de Janeiro", + "total": 1, + "url": "/data/airports/brazil/rj" + }, + { + "code": "SP", + "name": "Sao Paulo", + "total": 2, + "url": "/data/airports/brazil/sp" + } + ], + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/fixtures/airports_index.html b/python/tests/fixtures/airports_index.html new file mode 100644 index 0000000..a55ed2f --- /dev/null +++ b/python/tests/fixtures/airports_index.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/python/tests/fixtures/airports_state_rj.json b/python/tests/fixtures/airports_state_rj.json new file mode 100644 index 0000000..77d7a6b --- /dev/null +++ b/python/tests/fixtures/airports_state_rj.json @@ -0,0 +1,44 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "rj" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "RJ", + "name": "Rio de Janeiro", + "slug": "rj" + }, + "airports": [ + { + "id": 1252, + "name": "Rio de Janeiro Galeao International Airport", + "city": "Rio de Janeiro", + "iata": "GIG", + "icao": "SBGL", + "total": 2500 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/rj", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/fixtures/airports_state_sp.json b/python/tests/fixtures/airports_state_sp.json new file mode 100644 index 0000000..46c60c6 --- /dev/null +++ b/python/tests/fixtures/airports_state_sp.json @@ -0,0 +1,52 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "sp" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "SP", + "name": "Sao Paulo", + "slug": "sp" + }, + "airports": [ + { + "id": 1255, + "name": "Sao Paulo Guarulhos International Airport", + "city": "Sao Paulo", + "iata": "GRU", + "icao": "SBGR", + "total": 6796 + }, + { + "id": 1254, + "name": "Sao Paulo Congonhas Airport", + "city": "Sao Paulo", + "iata": "CGH", + "icao": "SBSP", + "total": 3100 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/sp", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/test_airports_offline.py b/python/tests/test_airports_offline.py new file mode 100644 index 0000000..39588f0 --- /dev/null +++ b/python/tests/test_airports_offline.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +"""Offline tests for the page crawl behind ``get_airports()``. + +FR24 lists airports one Inertia page per country, and per state for the largest +countries. These tests pin down which pages the crawl asks for and how it reads +them, against the bundled fixtures, so the PR gate covers the orchestration +without reaching FR24. +""" + +import json +import os +from typing import Any, Dict, List, Optional, Union + +from FlightRadarAPI import Countries, FlightRadar24API +from FlightRadarAPI.core import Core + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") + + +def _load(name: str) -> bytes: + with open(os.path.join(FIXTURES, name), "rb") as f: + return f.read() + + +# Every page the fixtures model, by the path FR24 serves it at. +PAGES: Dict[str, str] = { + "/data/airports": "airports_index.html", + "/data/airports/uruguay": "airports_country.json", + "/data/airports/brazil": "airports_country_states.json", + "/data/airports/brazil/sp": "airports_state_sp.json", + "/data/airports/brazil/rj": "airports_state_rj.json", + "/data/airports/myanmar-burma": "airports_country.html", +} + + +class _FakeResponse: + def __init__(self, status_code: int, content: Union[Dict[str, Any], bytes]) -> None: + self._status_code = status_code + self._content = content + + def get_status_code(self) -> int: + return self._status_code + + def get_content(self) -> Union[Dict[str, Any], bytes]: + return self._content + + def get_bytes_content(self) -> bytes: + if not isinstance(self._content, bytes): + raise ValueError("Expected bytes response, got JSON") + return self._content + + +class _FakeClient: + """Serves the fixture pages the way FR24 does. + + A request carrying the inertia headers gets the page JSON when the version + matches and a 409 otherwise; a plain request gets the HTML. The index page + only exists as HTML, like the real one as the crawl fetches it. + """ + + def __init__(self, version: Optional[str] = "919b94d2a33f365a34f99828f6eeec6a") -> None: + self.version = version + self.calls: List[Dict[str, Any]] = [] + + def _serve(self, url: str, **kwargs: Any) -> _FakeResponse: + self.calls.append({"url": url, **kwargs}) + path = url.replace(Core.flightradar_base_url, "") + fixture = PAGES.get(path, "airports_index.html") + headers = kwargs.get("headers") or {} + + if headers.get("x-inertia") == "true": + if headers.get("x-inertia-version") != self.version: + assert 409 in (kwargs.get("allowed_error_codes") or []), "a 409 must not raise" + return _FakeResponse(409, b"") + page = json.loads(_load(fixture)) if fixture.endswith(".json") else _page_of(_load(fixture)) + return _FakeResponse(200, page) + + if fixture.endswith(".json"): + fixture = fixture.replace(".json", ".html") + if not os.path.exists(os.path.join(FIXTURES, fixture)): + fixture = "airports_country.html" + return _FakeResponse(200, _load(fixture)) + + request = _serve + request_standalone = _serve + + +def _page_of(html: bytes) -> Dict[str, Any]: + from FlightRadarAPI.parsers import parse_inertia_page + page = parse_inertia_page(html) + assert page is not None + return page + + +def _api(client: _FakeClient) -> FlightRadar24API: + api = FlightRadar24API() + # Name-mangled private attribute -- set it the way Python stores it. + api._FlightRadar24API__client = client # type: ignore[attr-defined] + return api + + +def _paths(client: _FakeClient) -> List[str]: + return [call["url"].replace(Core.flightradar_base_url, "") for call in client.calls] + + +def test_get_airports_crawls_the_country_then_its_states(): + client = _FakeClient() + + airports = _api(client).get_airports([Countries.BRAZIL]) + + assert [a.iata for a in airports] == ["GIG", "GRU", "CGH"] + assert {a.country for a in airports} == {"Brazil"} + assert _paths(client) == ["/data/airports", "/data/airports/brazil", "/data/airports/brazil/rj", "/data/airports/brazil/sp"] + + +def test_get_airports_asks_for_the_page_json_with_the_index_version(): + client = _FakeClient() + + _api(client).get_airports(Countries.URUGUAY) + + index, country = client.calls + assert "x-inertia" not in index["headers"] + assert country["headers"]["x-inertia"] == "true" + assert country["headers"]["x-inertia-version"] == "919b94d2a33f365a34f99828f6eeec6a" + + +def test_get_airports_falls_back_to_html_when_the_assets_were_redeployed(): + client = _FakeClient(version="something-newer") + + airports = _api(client).get_airports(["uruguay"]) + + # The index answered with a stale version, the JSON request got a 409 and + # the HTML page was read instead. + assert len(airports) == 1 + assert _paths(client) == ["/data/airports", "/data/airports/uruguay", "/data/airports/uruguay"] + assert client.calls[2]["headers"] == Core.html_headers + + +def test_get_airports_without_countries_walks_the_whole_index(): + client = _FakeClient() + + airports = _api(client).get_airports() + + assert [a.iata for a in airports] == ["GIG", "GRU", "CGH", "RGN", "CYR", "MVD"] + assert _paths(client) == [ + "/data/airports", + "/data/airports/brazil", "/data/airports/myanmar-burma", "/data/airports/uruguay", + "/data/airports/brazil/rj", "/data/airports/brazil/sp", + ] + + +def test_get_airports_accepts_mixed_spellings_and_drops_repeats(): + client = _FakeClient() + + airports = _api(client).get_airports(["Uruguay", Countries.URUGUAY, "uruguay"]) + + assert [a.iata for a in airports] == ["CYR", "MVD"] + assert _paths(client) == ["/data/airports", "/data/airports/uruguay"] + + +def test_get_airports_with_an_empty_filter_makes_no_request(): + client = _FakeClient() + + assert _api(client).get_airports([]) == [] + assert _api(client).get_airports(iter([])) == [] + assert client.calls == [] + + +def test_get_airports_unknown_country_returns_nothing_and_warns(caplog): + client = _FakeClient() + + with caplog.at_level("WARNING", logger="FlightRadarAPI.api"): + airports = _api(client).get_airports(["atlantis"]) + + assert airports == [] + assert "no airports found for atlantis" in caplog.text diff --git a/python/tests/test_parsers_offline.py b/python/tests/test_parsers_offline.py index cf675ac..133b62b 100644 --- a/python/tests/test_parsers_offline.py +++ b/python/tests/test_parsers_offline.py @@ -9,28 +9,15 @@ """ import json -import math import os -import pytest - from FlightRadarAPI.core import Countries -from FlightRadarAPI.parsers import country_to_slug, parse_airlines_html, parse_airports_json +from FlightRadarAPI.parsers import ( + country_to_slug, parse_airlines_html, parse_airports_index, parse_airports_page, parse_inertia_page, +) FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") -# Keep in step with the same list in nodejs/tests/testParsersOffline.js. -NUMERIC_CASES = [ - ("whitespace", " ", None), - ("an array holding a number", [43], None), - ("a coordinate with a hemisphere suffix", "43.30 N", None), - ("an exponent overflowing a double", "1e999", None), - ("400 plain digits", "1" * 400, None), - ("a whole number as a string", "2436", 2436), - ("a negative decimal", "-23.4", -23.4), - ("a genuine zero", 0, 0), -] - def _load(name: str) -> bytes: with open(os.path.join(FIXTURES, name), "rb") as f: @@ -68,187 +55,135 @@ def test_parse_airlines_html_parses_aircraft_count(): by_name = {a["Name"]: a for a in airlines} assert by_name["LATAM Airlines"]["n_aircrafts"] == 340 assert by_name["Gol"]["n_aircrafts"] == 140 + # Thousands separator, as the page prints large fleets. + assert by_name["Delta Air Lines"]["n_aircrafts"] == 1900 + # Only the "Largest fleets" section carries a count. + assert by_name["Sky2"]["n_aircrafts"] is None -def test_parse_airlines_html_skips_invalid_rows(): +def test_parse_airlines_html_merges_an_airline_listed_in_two_sections(): airlines = parse_airlines_html(_load("airlines.html")) - # 5 valid rows; 2 invalid (no notranslate, wrong href) must be skipped. - assert len(airlines) == 5 - - -def test_parse_airlines_html_empty_input_returns_empty_list(): - assert parse_airlines_html(b"") == [] - assert parse_airlines_html(b"

no tbody here

") == [] - -# --- parse_airports_json --- - - -def test_parse_airports_json_extracts_basic_fields(): - airports = parse_airports_json(_load("airports.json")) - by_iata = {a.iata: a for a in airports} - assert "GRU" in by_iata and "GIG" in by_iata - - gru = by_iata["GRU"] - assert gru.icao == "SBGR" - assert gru.country == "Brazil" - assert abs(gru.latitude - (-23.429991)) < 1e-6 - assert abs(gru.longitude - (-46.4674)) < 1e-6 - assert gru.altitude == 2436 + # Delta and Gol appear under "Largest fleets" and again under "All airlines". + assert [a["Name"] for a in airlines].count("Delta Air Lines") == 1 + assert [a["Name"] for a in airlines].count("Gol") == 1 -def test_parse_airports_json_keeps_every_country_without_filter(): - countries = {a.country for a in parse_airports_json(_load("airports.json"))} - assert {"Brazil", "United States", "Spain"} <= countries - - -def test_parse_airports_json_filters_by_country_slug(): - airports = parse_airports_json(_load("airports.json"), ["united-states"]) - assert airports - assert all(a.country == "United States" for a in airports) - - -def test_parse_airports_json_accepts_several_countries(): - airports = parse_airports_json(_load("airports.json"), ["brazil", "spain"]) - assert {a.country for a in airports} == {"Brazil", "Spain"} +def test_parse_airlines_html_keeps_a_code_shaped_name(): + airlines = parse_airlines_html(_load("airlines.html")) + by_name = {a["Name"]: a for a in airlines} + assert by_name["TAP"] == {"Name": "TAP", "ICAO": "TAP", "IATA": "TP", "n_aircrafts": None} -def test_parse_airports_json_invalid_coordinates_become_none(): - """Regression: invalid coords used to be silently coerced to (0.0, 0.0) - placing the airport in the Gulf of Guinea. They must be None now.""" - airports = parse_airports_json(_load("airports.json")) - bad = next(a for a in airports if a.iata == "BAD") - assert bad.latitude is None - assert bad.longitude is None +def test_parse_airlines_html_skips_invalid_rows(): + airlines = parse_airlines_html(_load("airlines.html")) + # 6 airlines; the section link, the entry with no link, the wrong href and + # the one-letter name must be skipped. + assert len(airlines) == 6 + assert "All" not in [a["Name"] for a in airlines] -def test_parse_airports_json_rejects_numeric_looking_junk(): - payload = json.dumps({"rows": [{ - "name": "Junk Airport", "iata": "JNK", "icao": "JJNK", - "lat": "43.30 N", "lon": -8.37725, "country": "Spain", "alt": "-1", - }]}) - airport = parse_airports_json(payload)[0] +def test_parse_airlines_html_empty_input_returns_empty_list(): + assert parse_airlines_html(b"") == [] + assert parse_airlines_html(b"

no airlines here

") == [] - # One bad coordinate drops both; altitude is independent and survives. - assert airport.latitude is None - assert airport.longitude is None - assert airport.altitude == -1 - assert isinstance(airport.altitude, int) +# --- parse_inertia_page --- -@pytest.mark.parametrize("value,expected", [ - ("٤٣", None), # Arabic-Indic digits, which \d matched before re.ASCII - ("43", None), # U+FEFF, which String.trim() drops and str.strip() keeps - ("43", None), # U+001C, which str.strip() drops and String.trim() keeps - (" 43 ", 43), # plain spaces are still trimmed -]) -def test_parse_airports_json_agrees_with_node_on_unicode(value, expected): - payload = json.dumps({"rows": [{ - "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", - "lat": value, "lon": value, "alt": value, - }]}) +def test_parse_inertia_page_reads_the_data_page_attribute(): + page = parse_inertia_page(_load("airports_index.html")) + assert page["component"] == "Data/Airports" + assert page["version"] == "919b94d2a33f365a34f99828f6eeec6a" + assert "airportsByCountry" in page["props"] - assert parse_airports_json(payload)[0].altitude == expected +def test_parse_inertia_page_accepts_a_decoded_json_body(): + body = json.loads(_load("airports_country.json")) + assert parse_inertia_page(body) is body -@pytest.mark.parametrize( - "value,expected", - [(case[1], case[2]) for case in NUMERIC_CASES], - ids=[case[0] for case in NUMERIC_CASES], -) -def test_parse_airports_json_numeric_coercion(value, expected): - payload = json.dumps({"rows": [{ - "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", - "lat": value, "lon": value, "alt": value, - }]}) - airport = parse_airports_json(payload)[0] - assert airport.latitude == expected - assert airport.longitude == expected - assert airport.altitude == expected +def test_parse_inertia_page_rejects_unrecognised_payloads(): + assert parse_inertia_page(b"") is None + assert parse_inertia_page(b"
") is None + assert parse_inertia_page(b"
") is None + assert parse_inertia_page({"component": "Data/Airports"}) is None -@pytest.mark.parametrize("literal", ["1" * 400, "9007199254740993", "-" + "1" * 400]) -def test_parse_airports_json_survives_raw_numbers_no_double_can_hold(literal): - """Written as text: json.loads turns these into unbounded ints.""" - payload = ( - '{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",' - f'"lat":{literal},"lon":1,"alt":2}}]}}' - ) - latitude = parse_airports_json(payload)[0].latitude +# --- parse_airports_index --- - assert latitude is None or math.isfinite(latitude) +def test_parse_airports_index_lists_every_country_with_its_slug(): + countries = parse_airports_index(_load("airports_index.html")) + assert [c["slug"] for c in countries] == ["brazil", "myanmar-burma", "uruguay"] + assert countries[1]["name"] == "MYANMAR (BURMA)" -@pytest.mark.parametrize("literal", ["null", "0", "false", "true", "[]", "{}", "123"]) -def test_parse_airports_json_keeps_text_fields_as_strings(literal): - """Anything else breaks get_country_flag(airport.country).""" - payload = ( - '{"rows":[{"name":%s,"iata":%s,"icao":%s,"country":%s,"lat":1,"lon":2,"alt":3}]}' - % (literal, literal, literal, literal) - ) - airport = parse_airports_json(payload)[0] +def test_parse_airports_index_slugs_match_the_countries_enum(): + slugs = {c["slug"] for c in parse_airports_index(_load("airports_index.html"))} + assert slugs <= {c.value for c in Countries} - assert [airport.name, airport.iata, airport.icao, airport.country] == ["", "", "", ""] +def test_parse_airports_index_invalid_payload_returns_empty_list(): + assert parse_airports_index(b"") == [] + assert parse_airports_index({"props": {}}) == [] + assert parse_airports_index({"props": {"airportsByCountry": [1, "x", {"name": None}]}}) == [] -def test_parse_airports_json_country_spelling_and_slug_round_trip(): - ann = next(a for a in parse_airports_json(_load("airports.json")) if a.iata == "VBA") - assert ann.country == "Myanmar (Burma)" - assert country_to_slug(ann.country) == "myanmar-burma" - assert ann.altitude == 43 +# --- parse_airports_page --- +def test_parse_airports_page_extracts_the_airports_of_a_country(): + airports, state_paths = parse_airports_page(json.loads(_load("airports_country.json"))) + assert state_paths == [] + assert [a.iata for a in airports] == ["CYR", "MVD"] -def test_parse_airports_json_filter_accepts_either_country_spelling(): - for slug in ["myanmar-burma", "Myanmar (Burma)"]: - assert len(parse_airports_json(_load("airports.json"), [slug])) == 1 + mvd = airports[1] + assert mvd.name == "Montevideo Carrasco International Airport" + assert mvd.icao == "SUMU" + assert mvd.country == "Uruguay" -def test_parse_airports_json_keeps_altitude_as_int(): - """Both ports must agree on altitude: 2436, never 2436.0.""" - gru = next(a for a in parse_airports_json(_load("airports.json")) if a.iata == "GRU") - assert isinstance(gru.altitude, int) +def test_parse_airports_page_listing_carries_no_position(): + airports, _ = parse_airports_page(json.loads(_load("airports_country.json"))) + assert all(a.latitude is None and a.longitude is None and a.altitude is None for a in airports) -def test_parse_airports_json_skips_rows_that_are_not_objects(): - payload = ( - '{"rows":[[1,2,3],"x",7,null,' - '{"name":"Real","iata":"RRR","icao":"RRRR","country":"Spain","lat":1,"lon":2,"alt":3}]}' - ) +def test_parse_airports_page_returns_the_state_pages_of_a_split_country(): + airports, state_paths = parse_airports_page(json.loads(_load("airports_country_states.json"))) + assert airports == [] + assert state_paths == ["/data/airports/brazil/rj", "/data/airports/brazil/sp"] - assert [a.iata for a in parse_airports_json(payload)] == ["RRR"] +def test_parse_airports_page_state_page_keeps_the_country_name(): + airports, state_paths = parse_airports_page(json.loads(_load("airports_state_sp.json"))) + assert state_paths == [] + assert [a.iata for a in airports] == ["GRU", "CGH"] + assert {a.country for a in airports} == {"Brazil"} -def test_parse_airports_json_unknown_country_returns_empty_list(): - assert parse_airports_json(_load("airports.json"), ["atlantis"]) == [] +def test_parse_airports_page_reads_an_html_page(): + airports, _ = parse_airports_page(_load("airports_country.html")) + assert [a.iata for a in airports] == ["RGN"] + # FR24 spells it "Myanmar (Burma)" on the page and "myanmar-burma" in URLs. + assert airports[0].country == "Myanmar (Burma)" + assert country_to_slug(airports[0].country) == Countries.MYANMAR_BURMA.value -def test_parse_airports_json_invalid_payload_returns_empty_list(): - assert parse_airports_json(b"") == [] - assert parse_airports_json(b"{}") == [] - assert parse_airports_json(b"not json") == [] +def test_parse_airports_page_skips_rows_that_are_not_objects(): + payload = {"props": {"country": {"name": "Nowhere"}, "airports": [1, "x", None, {"iata": "RRR"}]}} + airports, _ = parse_airports_page(payload) + assert [a.iata for a in airports] == ["RRR"] -# --- country_to_slug --- -def test_country_to_slug_matches_countries_enum_spelling(): - assert country_to_slug("United States") == "united-states" - assert country_to_slug("Democratic Republic Of The Congo") == "democratic-republic-of-the-congo" - assert country_to_slug("Curacao") == "curacao" - assert country_to_slug("Curaçao") == "curacao" - assert country_to_slug(None) == "" +def test_parse_airports_page_keeps_text_fields_as_strings(): + payload = {"props": {"country": {"name": ["Brazil"]}, "airports": [{"name": 42, "iata": None, "icao": {"x": 1}}]}} + airport = parse_airports_page(payload)[0][0] + assert (airport.name, airport.iata, airport.icao, airport.country) == ("", "", "", "") -def test_country_to_slug_takes_a_countries_member(): - """get_airports accepts them, so get_country_flag(airport.country) style calls - must not slugify "Countries.BRAZIL".""" - assert country_to_slug(Countries.BRAZIL) == "brazil" - assert country_to_slug(Countries.MYANMAR_BURMA) == "myanmar-burma" +def test_parse_airports_page_ignores_malformed_states(): + payload = {"props": {"states": [{"url": "/data/airports/x/a"}, {"url": ""}, {"code": "B"}, "junk", {"url": 3}]}} + assert parse_airports_page(payload) == ([], ["/data/airports/x/a"]) -def test_country_to_slug_strips_parentheses_for_flag_urls(): - """These 404'd while the slug was a plain space-to-hyphen replacement.""" - assert country_to_slug("Myanmar (Burma)") == "myanmar-burma" - assert country_to_slug("Cocos (Keeling) Islands") == "cocos-keeling-islands" - assert country_to_slug("Falkland Islands (Malvinas)") == "falkland-islands-malvinas" - assert country_to_slug("Timor-Leste (East Timor)") == "timor-leste-east-timor" +def test_parse_airports_page_invalid_payload_returns_nothing(): + assert parse_airports_page(b"") == ([], []) + assert parse_airports_page(b"not a data page") == ([], []) + assert parse_airports_page({"props": {}}) == ([], []) diff --git a/python/tests/util.py b/python/tests/util.py index 3c03b51..8b887d7 100644 --- a/python/tests/util.py +++ b/python/tests/util.py @@ -12,13 +12,28 @@ def raise_multiple(errors: List) -> None: finally: raise_multiple(errors) -def repeat_test(attempts: int, after: int, errors: Optional[Sequence[Type[BaseException]]] = None) -> Callable: +def is_rate_limited(error: BaseException) -> bool: + """ + Whether FR24 answered 429: shared CI runners trip its rate limit, and a + pause is all that fixes it. + """ + response = getattr(error, "response", None) + return getattr(response, "status_code", None) == 429 or "429" in str(error).split(":")[0] + + +def repeat_test( + attempts: int, + after: int, + errors: Optional[Sequence[Type[BaseException]]] = None, + retry_on: Optional[Callable[[BaseException], bool]] = is_rate_limited, +) -> Callable: """ Decorator to repeat a test N times for specific errors. :param attempts: Number of attempts for testing :param after: Time in seconds to wait for each attempt :param errors: Sequence of exception classes to retry on; if None, retry on any error + :param retry_on: Predicate that also marks an error as retryable, whatever its class """ if errors is not None: invalid = [e for e in errors if not (isinstance(e, type) and issubclass(e, BaseException))] @@ -34,7 +49,9 @@ def wrapper(*args, **kwargs): return test_function(*args, **kwargs) except Exception as error: - if errors is not None and not isinstance(error, tuple(errors)): raise error + retryable = errors is None or isinstance(error, tuple(errors)) + if not retryable and retry_on is not None: retryable = retry_on(error) + if not retryable: raise error if after is not None: time.sleep(after) error_list.append(error)