From 98ae1ebf822450e616a3d3fc1941e9956116057b Mon Sep 17 00:00:00 2001 From: JeanExtreme002 Date: Mon, 9 Nov 2026 02:03:09 -0300 Subject: [PATCH 1/2] fix: follow the FR24 airports and airlines page redesign The scheduled live runs of the three packages went red on 2026-11-09: FlightRadar24 redesigned its data pages. - `/_json/airports.php`, the one-payload airports feed, is gone (404). Airports are now listed on Inertia pages: `/data/airports` names every country and declares the asset version, `/data/airports/` lists the airports, or the states for the largest countries, and `/data/airports//` lists those. `get_airports()` now reads the index and crawls one page per country and state, `max_workers` at a time, asking for the page JSON with the Inertia headers and falling back to the HTML (whose data-page attribute carries the same object) on a 409. The listing no longer carries coordinates or altitude: those fields are None, and `get_airport(code)` has them. - `/data/airlines` is no longer a table: airlines are list entries with the name in a titled element, the codes as badges and the fleet size only under "Largest fleets". The parser follows the new markup and merges an airline the page lists in several sections. The offline fixtures model the new pages (shared by the three ports), the parser tests mirror each other, and a new offline suite pins the crawl: request order, version header, 409 fallback, grouping by country, deduplication and the unknown-country warning. The Python live tests also retry on HTTP 429, which the shared CI runners trip on the feed. --- docs/go.md | 6 + docs/nodejs.md | 6 + docs/python.md | 6 + go/flightradarapi/api.go | 222 +++++++- go/flightradarapi/api_test.go | 394 +++++++++++++- go/flightradarapi/core.go | 26 +- go/flightradarapi/parsers.go | 318 ++++++++---- go/flightradarapi/parsers_test.go | 485 +++++++++--------- go/flightradarapi/testdata/airlines.html | 132 ++--- go/flightradarapi/testdata/airports.json | 77 --- .../testdata/airports_country.html | 8 + .../testdata/airports_country.json | 51 ++ .../testdata/airports_country_states.json | 41 ++ .../testdata/airports_index.html | 8 + .../testdata/airports_state_rj.json | 44 ++ .../testdata/airports_state_sp.json | 52 ++ nodejs/FlightRadarAPI/api.js | 98 +++- nodejs/FlightRadarAPI/core.js | 12 +- nodejs/FlightRadarAPI/index.d.ts | 7 +- nodejs/FlightRadarAPI/parsers.js | 309 ++++++----- nodejs/package.json | 2 +- nodejs/tests/fixtures/airlines.html | 132 ++--- nodejs/tests/fixtures/airports.json | 77 --- nodejs/tests/fixtures/airports_country.html | 8 + nodejs/tests/fixtures/airports_country.json | 51 ++ .../fixtures/airports_country_states.json | 41 ++ nodejs/tests/fixtures/airports_index.html | 8 + nodejs/tests/fixtures/airports_state_rj.json | 44 ++ nodejs/tests/fixtures/airports_state_sp.json | 52 ++ nodejs/tests/testAirportsOffline.js | 202 ++++++++ nodejs/tests/testParsersOffline.js | 247 ++++----- python/FlightRadarAPI/api.py | 90 +++- python/FlightRadarAPI/core.py | 14 +- python/FlightRadarAPI/parsers.py | 281 +++++----- python/tests/fixtures/airlines.html | 132 ++--- python/tests/fixtures/airports.json | 77 --- python/tests/fixtures/airports_country.html | 8 + python/tests/fixtures/airports_country.json | 51 ++ .../fixtures/airports_country_states.json | 41 ++ python/tests/fixtures/airports_index.html | 8 + python/tests/fixtures/airports_state_rj.json | 44 ++ python/tests/fixtures/airports_state_sp.json | 52 ++ python/tests/test_airports_offline.py | 176 +++++++ python/tests/test_parsers_offline.py | 249 ++++----- python/tests/util.py | 21 +- 45 files changed, 3028 insertions(+), 1382 deletions(-) delete mode 100644 go/flightradarapi/testdata/airports.json create mode 100644 go/flightradarapi/testdata/airports_country.html create mode 100644 go/flightradarapi/testdata/airports_country.json create mode 100644 go/flightradarapi/testdata/airports_country_states.json create mode 100644 go/flightradarapi/testdata/airports_index.html create mode 100644 go/flightradarapi/testdata/airports_state_rj.json create mode 100644 go/flightradarapi/testdata/airports_state_sp.json delete mode 100644 nodejs/tests/fixtures/airports.json create mode 100644 nodejs/tests/fixtures/airports_country.html create mode 100644 nodejs/tests/fixtures/airports_country.json create mode 100644 nodejs/tests/fixtures/airports_country_states.json create mode 100644 nodejs/tests/fixtures/airports_index.html create mode 100644 nodejs/tests/fixtures/airports_state_rj.json create mode 100644 nodejs/tests/fixtures/airports_state_sp.json create mode 100644 nodejs/tests/testAirportsOffline.js delete mode 100644 python/tests/fixtures/airports.json create mode 100644 python/tests/fixtures/airports_country.html create mode 100644 python/tests/fixtures/airports_country.json create mode 100644 python/tests/fixtures/airports_country_states.json create mode 100644 python/tests/fixtures/airports_index.html create mode 100644 python/tests/fixtures/airports_state_rj.json create mode 100644 python/tests/fixtures/airports_state_sp.json create mode 100644 python/tests/test_airports_offline.py diff --git a/docs/go.md b/docs/go.md index 6997bfa..ce09a61 100644 --- a/docs/go.md +++ b/docs/go.md @@ -57,6 +57,12 @@ You can fetch various types of data using the following methods: allAirports, err := client.GetAirports(ctx, nil) ``` + FR24 lists airports one page per country (and per state for the largest + countries), so this makes one request per page: a handful for a few + countries, a few hundred for every airport, `MaxWorkers` at a time. The + listing carries the name, codes and country only; use `GetAirport(ctx, code)` + for the position and altitude of an airport. + - **Airlines list:** ```go diff --git a/docs/nodejs.md b/docs/nodejs.md index 8fc3264..b1cfca1 100644 --- a/docs/nodejs.md +++ b/docs/nodejs.md @@ -36,6 +36,12 @@ You can fetch various types of data using the following methods: let airports = await frApi.getAirports(...); // Returns a list of Airport objects ``` + FR24 lists airports one page per country (and per state for the largest + countries), so this makes one request per page: a handful for a few + countries, a few hundred for every airport, `maxWorkers` at a time. The + listing carries the name, codes and country only; use `getAirport(code)` + for the position and altitude of an airport. + - **Airlines list:** ```javascript diff --git a/docs/python.md b/docs/python.md index e7a3fc7..d9ec6a8 100644 --- a/docs/python.md +++ b/docs/python.md @@ -36,6 +36,12 @@ You can fetch various types of data using the following methods: airports = fr_api.get_airports(...) # Returns a list of Airport objects ``` + FR24 lists airports one page per country (and per state for the largest + countries), so this makes one request per page: a handful for a few + countries, a few hundred for every airport, `max_workers` at a time. The + listing carries the name, codes and country only; use `get_airport(code)` + for the position and altitude of an airport. + - **Airlines list:** ```python diff --git a/go/flightradarapi/api.go b/go/flightradarapi/api.go index 19e1d3d..0215037 100644 --- a/go/flightradarapi/api.go +++ b/go/flightradarapi/api.go @@ -9,6 +9,7 @@ import ( "math" "net/http" "net/url" + "slices" "strings" "sync" "time" @@ -286,23 +287,234 @@ func (c *Client) GetAirportDisruptions(ctx context.Context) (map[string]any, err // GetAirports returns every airport, or only those of the given countries. Pass // nil for every airport; an empty non-nil slice selects none. +// +// FR24 lists airports one page per country, and one per state for the largest +// countries, so this fetches one page per country asked for (a few hundred for +// every airport), MaxWorkers at a time. The listing carries the name, codes and +// country only: use [Client.GetAirport] for the position and altitude of an +// airport. func (c *Client) GetAirports(ctx context.Context, countries []Country) ([]*Airport, error) { if countries != nil && len(countries) == 0 { return []*Airport{}, nil } - response, err := c.client.request(ctx, c.endpoints.airportsJSON, requestOptions{ - headers: jsonHeaders, + // The index names every country, and declares the asset version the JSON + // requests below have to quote. + response, err := c.client.request(ctx, c.endpoints.airportsData, requestOptions{ + headers: htmlHeaders, + timeout: c.Timeout, + }) + + if err != nil { + return nil, err + } + + indexPage, _ := parseInertiaPage(response.Body) + version := toText(indexPage["version"]) + + var slugs []string + + if countries == nil { + for _, country := range parseAirportsIndex(indexPage) { + slugs = append(slugs, country.Slug) + } + } else { + for _, country := range countries { + slugs = append(slugs, countryToSlug(string(country))) + } + } + + slugs = uniqueInOrder(slices.DeleteFunc(slugs, func(slug string) bool { return slug == "" })) + + countryURLs := make([]string, len(slugs)) + + for index, slug := range slugs { + countryURLs[index] = c.endpoints.airportsByCountryURL(slug) + } + + countryPages, err := c.getAirportPages(ctx, countryURLs, version) + + if err != nil { + return nil, err + } + + // Second pass for the countries split by state, every state at once. A + // state path is relative to the site, so it is resolved against the page + // that listed it rather than against a fixed host. + var stateURLs []string + var stateOwners []int + + for owner, page := range countryPages { + for _, path := range page.statePaths { + stateURLs = append(stateURLs, resolveReference(countryURLs[owner], path)) + stateOwners = append(stateOwners, owner) + } + } + + statePages, err := c.getAirportPages(ctx, stateURLs, version) + + if err != nil { + return nil, err + } + + // Grouped by country, in the order asked for, each country's states after + // it. + airportsByCountry := make([][]*Airport, len(countryPages)) + + for index, page := range countryPages { + airportsByCountry[index] = page.airports + } + for index, page := range statePages { + owner := stateOwners[index] + airportsByCountry[owner] = append(airportsByCountry[owner], page.airports...) + } + + airports := []*Airport{} + + for _, found := range airportsByCountry { + airports = append(airports, found...) + } + + // A country FR24 does not know answers with the index page, which lists + // neither airports nor states. + var missing []string + + for index, page := range countryPages { + if len(page.airports) == 0 && len(page.statePaths) == 0 { + missing = append(missing, slugs[index]) + } + } + if len(missing) > 0 { + log().Warn("GetAirports: no airports found for some countries — check the Country constants", + "countries", strings.Join(missing, ", ")) + } + return airports, nil +} + +// airportsPage is what one airports listing page said: its airports, or the +// paths of the state pages it is split into. +type airportsPage struct { + airports []*Airport + statePaths []string +} + +// getAirportPages fetches and parses the airport listing pages at urls, in +// order, MaxWorkers at a time. The first failure fails the whole call. +func (c *Client) getAirportPages(ctx context.Context, urls []string, version string) ([]airportsPage, error) { + pages := make([]airportsPage, len(urls)) + + if len(urls) == 0 { + return pages, nil + } + + // Clamped rather than skipped: MaxWorkers is public, and a zero there must + // not deadlock the crawl. + workers := make(chan struct{}, max(c.MaxWorkers, 1)) + var group sync.WaitGroup + var once sync.Once + var firstErr error + + for index, target := range urls { + workers <- struct{}{} + group.Add(1) + + go func() { + defer group.Done() + defer func() { <-workers }() + + payload, err := c.getInertiaPage(ctx, target, version) + + if err != nil { + once.Do(func() { firstErr = err }) + return + } + pages[index].airports, pages[index].statePaths = parseAirportsPage(payload) + }() + } + group.Wait() + + if firstErr != nil { + return nil, firstErr + } + return pages, nil +} + +// getInertiaPage fetches a FR24 data page as its Inertia page object, falling +// back to the HTML. It returns the decoded JSON body, or the HTML bytes, for +// parseInertiaPage to read. +// +// With the asset version FR24 answers the bare page JSON, a fraction of the +// HTML. A 409 means the assets were redeployed since the version was read: the +// HTML carries the same page object and needs no version. +func (c *Client) getInertiaPage(ctx context.Context, target, version string) (any, error) { + if version != "" { + response, err := c.client.requestStandalone(ctx, target, requestOptions{ + headers: withHeaders(inertiaHeaders, map[string]string{"x-inertia-version": version}), + allowedErrorCodes: []int{409}, + timeout: c.Timeout, + }) + + if err != nil { + return nil, err + } + + if response.StatusCode != 409 { + if !response.IsJSON() { + return response.Body, nil + } + + // Decoded as any value, not as an object: a body that is not a + // page object is the parser's warning, not a failed call. + var page any + + if err := json.Unmarshal([]byte(response.Text()), &page); err != nil { + return nil, fmt.Errorf("%w: could not parse the JSON body of %s: %w", ErrFlightRadar, response.URL, err) + } + return page, nil + } + } + + response, err := c.client.requestStandalone(ctx, target, requestOptions{ + headers: htmlHeaders, timeout: c.Timeout, }) if err != nil { return nil, err } + return response.Body, nil +} - // The raw body, not JSON(): an HTML body reaches the parser's guard, and a - // malformed JSON body still returns empty rather than failing the call. - return parseAirportsJSON(response.Body, countries), nil +// uniqueInOrder drops repeated values, keeping the first occurrence of each. +func uniqueInOrder(values []string) []string { + seen := make(map[string]bool, len(values)) + unique := values[:0:0] + + for _, value := range values { + if !seen[value] { + seen[value] = true + unique = append(unique, value) + } + } + return unique +} + +// resolveReference resolves a path FR24 printed on a page against that page's +// URL, so "/data/airports/brazil/sp" lands on the host the page came from. A +// reference that will not parse is returned as given. +func resolveReference(base, reference string) string { + baseURL, err := url.Parse(base) + + if err != nil { + return reference + } + + referenceURL, err := url.Parse(reference) + + if err != nil { + return reference + } + return baseURL.ResolveReference(referenceURL).String() } // GetBookmarks returns the bookmarks of the logged-in account. diff --git a/go/flightradarapi/api_test.go b/go/flightradarapi/api_test.go index b5449d8..31eca65 100644 --- a/go/flightradarapi/api_test.go +++ b/go/flightradarapi/api_test.go @@ -1,16 +1,22 @@ package flightradarapi import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "math" "net/http" "net/http/cookiejar" "net/http/httptest" "net/url" + "os" + "path/filepath" + "slices" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -48,7 +54,8 @@ func testEndpoints(base string) endpoints { historicalData: base + "/download/?flight=%s&file=%s&trailLimit=0&history=%d", apiAirportData: base + "/common/v1/airport.json", airportData: base + "/airports/traffic-stats/?airport=%s", - airportsJSON: base + "/_json/airports.php", + airportsData: base + "/data/airports", + airportsByCountry: base + "/data/airports/%s", airportDisruption: base + "/webapi/v1/airport-disruptions", airlinesData: base + "/data/airlines", volcanicEruptionData: base + "/weather/volcanic", @@ -97,54 +104,399 @@ func TestGetAirlinesParsesTheListingPage(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(airlines) != 5 { - t.Errorf("got %d airlines, want 5", len(airlines)) + if len(airlines) != 6 { + t.Errorf("got %d airlines, want 6", len(airlines)) } } -func TestGetAirportsFiltersByCountry(t *testing.T) { +// --- GetAirports --- +// +// FR24 lists airports one Inertia page per country, and per state for the +// largest countries. These tests pin down which pages the crawl asks for and +// how it reads them, against the bundled fixtures, mirroring +// python/tests/test_airports_offline.py. + +// Every page the fixtures model, by the path FR24 serves it at. +var airportPages = map[string]string{ + "/data/airports": "airports_index.html", + "/data/airports/uruguay": "airports_country.json", + "/data/airports/brazil": "airports_country_states.json", + "/data/airports/brazil/sp": "airports_state_sp.json", + "/data/airports/brazil/rj": "airports_state_rj.json", + "/data/airports/myanmar-burma": "airports_country.html", +} + +const fixtureAssetVersion = "919b94d2a33f365a34f99828f6eeec6a" + +// airportsCall is one request the fake FR24 saw. +type airportsCall struct { + path string + headers http.Header +} + +// airportsServer serves the fixture pages the way FR24 does. A request carrying +// the inertia headers gets the page JSON when the version matches and a 409 +// otherwise; a plain request gets the HTML. The index page only exists as HTML, +// like the real one as the crawl fetches it. +type airportsServer struct { + t *testing.T + version string + + mu sync.Mutex + calls []airportsCall +} + +func newAirportsServer(t *testing.T, version string) *airportsServer { + t.Helper() + return &airportsServer{t: t, version: version} +} + +func (s *airportsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.calls = append(s.calls, airportsCall{path: r.URL.Path, headers: r.Header.Clone()}) + s.mu.Unlock() + + fixture, ok := airportPages[r.URL.Path] + + if !ok { + fixture = "airports_index.html" + } + + if r.Header.Get("X-Inertia") == "true" { + if r.Header.Get("X-Inertia-Version") != s.version { + w.WriteHeader(http.StatusConflict) + return + } + if strings.HasSuffix(fixture, ".json") { + w.Header().Set("Content-Type", "application/json") + w.Write(loadFixture(s.t, fixture)) + return + } + + page, ok := parseInertiaPage(loadFixture(s.t, fixture)) + + if !ok { + s.t.Errorf("fixture %s carries no page object", fixture) + } + writeJSON(s.t, w, page) + return + } + + if strings.HasSuffix(fixture, ".json") { + fixture = strings.TrimSuffix(fixture, ".json") + ".html" + + if _, err := os.Stat(filepath.Join("testdata", fixture)); err != nil { + fixture = "airports_country.html" + } + } + w.Header().Set("Content-Type", "text/html") + w.Write(loadFixture(s.t, fixture)) +} + +// paths are the paths requested so far, in order. +func (s *airportsServer) paths() []string { + s.mu.Lock() + defer s.mu.Unlock() + paths := make([]string, 0, len(s.calls)) + + for _, call := range s.calls { + paths = append(paths, call.path) + } + return paths +} + +func (s *airportsServer) call(index int) airportsCall { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls[index] +} + +// newSerialAirportsClient wires a client to the fake, one request at a time, so +// the order of the requests is the order of the crawl rather than the order the +// goroutines happened to run in. +func newSerialAirportsClient(t *testing.T, server *airportsServer) *Client { + t.Helper() + client := newTestClient(t, server) + client.MaxWorkers = 1 + return client +} + +func TestGetAirportsCrawlsTheCountryThenItsStates(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + + airports, err := newSerialAirportsClient(t, server).GetAirports(context.Background(), []Country{CountryBrazil}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := iatas(airports); !slices.Equal(got, []string{"GIG", "GRU", "CGH"}) { + t.Errorf("got %v, want GIG, GRU, CGH", got) + } + for _, airport := range airports { + if airport.Country != "Brazil" { + t.Errorf("%s: got country %q, want Brazil", airport.IATA, airport.Country) + } + } + + want := []string{"/data/airports", "/data/airports/brazil", "/data/airports/brazil/rj", "/data/airports/brazil/sp"} + + if got := server.paths(); !slices.Equal(got, want) { + t.Errorf("got requests %v, want %v", got, want) + } +} + +func TestGetAirportsAsksForThePageJSONWithTheIndexVersion(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + + if _, err := newTestClient(t, server).GetAirports(context.Background(), []Country{CountryUruguay}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := server.paths(); !slices.Equal(got, []string{"/data/airports", "/data/airports/uruguay"}) { + t.Fatalf("got requests %v, want the index then the country", got) + } + + index, country := server.call(0), server.call(1) + + if index.headers.Get("X-Inertia") != "" || !strings.Contains(index.headers.Get("Accept"), "text/html") { + t.Errorf("index: got headers %v, want the html headers", index.headers) + } + if country.headers.Get("X-Inertia") != "true" || country.headers.Get("X-Inertia-Version") != fixtureAssetVersion { + t.Errorf("country: got headers %v, want the inertia headers with the index version", country.headers) + } + if country.headers.Get("X-Requested-With") != "XMLHttpRequest" || country.headers.Get("Accept") != "application/json" { + t.Errorf("country: got headers %v, want an XMLHttpRequest for JSON", country.headers) + } +} + +func TestGetAirportsFallsBackToHTMLWhenTheAssetsWereRedeployed(t *testing.T) { + server := newAirportsServer(t, "something-newer") + + airports, err := newTestClient(t, server).GetAirports(context.Background(), []Country{"uruguay"}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // The index answered with a stale version, the JSON request got a 409 and + // the HTML page was read instead. + if len(airports) != 1 { + t.Errorf("got %d airports, want the one on the HTML page", len(airports)) + } + + want := []string{"/data/airports", "/data/airports/uruguay", "/data/airports/uruguay"} + + if got := server.paths(); !slices.Equal(got, want) { + t.Fatalf("got requests %v, want %v", got, want) + } + + retry := server.call(2).headers + + if retry.Get("X-Inertia") != "" || !strings.Contains(retry.Get("Accept"), "text/html") { + t.Errorf("got headers %v, want the html headers", retry) + } +} + +func TestGetAirportsReadsTheHTMLWhenTheIndexDeclaresNoVersion(t *testing.T) { + // An index with no page object leaves the crawl without a version, so it + // asks for the HTML straight away rather than fishing for a 409. mux := http.NewServeMux() - mux.HandleFunc("/_json/airports.php", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.Write(loadFixture(t, "airports.json")) + server := newAirportsServer(t, fixtureAssetVersion) + mux.HandleFunc("/data/airports", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte("maintenance")) }) + mux.Handle("/data/airports/", server) - client := newTestClient(t, mux) - airports, err := client.GetAirports(context.Background(), []Country{CountryBrazil}) + airports, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryMyanmarBurma}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := iatas(airports); !slices.Equal(got, []string{"RGN"}) { + t.Errorf("got %v, want RGN", got) + } + if got := server.paths(); !slices.Equal(got, []string{"/data/airports/myanmar-burma"}) { + t.Fatalf("got requests %v, want the country page once", got) + } + if headers := server.call(0).headers; headers.Get("X-Inertia") != "" { + t.Errorf("got headers %v, want the html headers", headers) + } +} + +func TestGetAirportsWithoutCountriesWalksTheWholeIndex(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + + airports, err := newSerialAirportsClient(t, server).GetAirports(context.Background(), nil) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(airports) != 4 { - t.Errorf("got %d Brazilian airports, want 4", len(airports)) + if got := iatas(airports); !slices.Equal(got, []string{"GIG", "GRU", "CGH", "RGN", "CYR", "MVD"}) { + t.Errorf("got %v, want every airport grouped by country", got) + } + + want := []string{ + "/data/airports", + "/data/airports/brazil", "/data/airports/myanmar-burma", "/data/airports/uruguay", + "/data/airports/brazil/rj", "/data/airports/brazil/sp", + } + + if got := server.paths(); !slices.Equal(got, want) { + t.Errorf("got requests %v, want %v", got, want) + } +} + +func TestGetAirportsGroupsByCountryWhateverOrderThePagesArriveIn(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) + client := newTestClient(t, server) + client.MaxWorkers = 16 + + // Repeated so a scheduling order that happens to match does not pass it. + for range 5 { + airports, err := client.GetAirports(context.Background(), []Country{CountryUruguay, CountryBrazil, CountryMyanmarBurma}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := iatas(airports); !slices.Equal(got, []string{"CYR", "MVD", "GIG", "GRU", "CGH", "RGN"}) { + t.Fatalf("got %v, want the countries in the order asked for", got) + } } +} + +func TestGetAirportsAcceptsMixedSpellingsAndDropsRepeats(t *testing.T) { + server := newAirportsServer(t, fixtureAssetVersion) - every, err := client.GetAirports(context.Background(), nil) + airports, err := newTestClient(t, server).GetAirports(context.Background(), + []Country{"Uruguay", CountryUruguay, "uruguay", ""}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(every) != 8 { - t.Errorf("got %d airports for nil, want all 8", len(every)) + if got := iatas(airports); !slices.Equal(got, []string{"CYR", "MVD"}) { + t.Errorf("got %v, want CYR and MVD once", got) + } + if got := server.paths(); !slices.Equal(got, []string{"/data/airports", "/data/airports/uruguay"}) { + t.Errorf("got requests %v, want the country page once", got) } } func TestGetAirportsWithAnEmptyFilterMakesNoRequest(t *testing.T) { - var calls atomic.Int32 + server := newAirportsServer(t, fixtureAssetVersion) + + airports, err := newTestClient(t, server).GetAirports(context.Background(), []Country{}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if airports == nil || len(airports) != 0 || len(server.paths()) != 0 { + t.Errorf("got %v after %v, want an empty slice and no request", airports, server.paths()) + } +} + +func TestGetAirportsUnknownCountryReturnsNothingAndWarns(t *testing.T) { + var logged bytes.Buffer + SetLogger(slog.New(slog.NewTextHandler(&logged, nil))) + t.Cleanup(func() { SetLogger(nil) }) + + server := newAirportsServer(t, fixtureAssetVersion) + airports, err := newTestClient(t, server).GetAirports(context.Background(), []Country{"atlantis"}) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if airports == nil || len(airports) != 0 { + t.Errorf("got %v, want an empty slice", airports) + } + + output := logged.String() + + if !strings.Contains(output, "level=WARN") || !strings.Contains(output, "no airports found") || + !strings.Contains(output, "atlantis") { + t.Errorf("got %q, want a warning naming atlantis", output) + } +} + +func TestGetAirportsFailsWhenAPageCannotBeFetched(t *testing.T) { mux := http.NewServeMux() - mux.HandleFunc("/_json/airports.php", func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) + server := newAirportsServer(t, fixtureAssetVersion) + mux.Handle("/", server) + mux.HandleFunc("/data/airports/brazil/sp", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + + _, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryBrazil}) + var statusErr *StatusError + + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusInternalServerError { + t.Fatalf("got %v, want the 500 reported", err) + } +} + +func TestGetAirportsFailsWhenTheIndexCannotBeFetched(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/data/airports", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }) + + if _, err := newTestClient(t, mux).GetAirports(context.Background(), nil); err == nil { + t.Fatal("got no error, want the 503 reported") + } +} + +func TestGetAirportsRejectsAMalformedPageJSON(t *testing.T) { + mux := http.NewServeMux() + server := newAirportsServer(t, fixtureAssetVersion) + mux.Handle("/", server) + mux.HandleFunc("/data/airports/uruguay", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - w.Write(loadFixture(t, "airports.json")) + w.Write([]byte(`{"component": "Data/AirportsByCountry", "props": `)) }) - airports, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{}) + _, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryUruguay}) + + if !errors.Is(err, ErrFlightRadar) || !strings.Contains(err.Error(), "could not parse the JSON body") { + t.Fatalf("got %v, want the malformed body reported", err) + } +} + +func TestGetAirportsTreatsAPageJSONThatIsNotAnObjectAsEmpty(t *testing.T) { + mux := http.NewServeMux() + server := newAirportsServer(t, fixtureAssetVersion) + mux.Handle("/", server) + mux.HandleFunc("/data/airports/uruguay", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`null`)) + }) + + airports, err := newTestClient(t, mux).GetAirports(context.Background(), []Country{CountryUruguay}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(airports) != 0 || calls.Load() != 0 { - t.Errorf("got %d airports after %d calls, want neither", len(airports), calls.Load()) + if len(airports) != 0 { + t.Errorf("got %d airports, want none", len(airports)) + } +} + +func TestGetAirportsResolvesStatePathsAgainstTheCountryPage(t *testing.T) { + // FR24 prints state paths without a host; an absolute URL must survive too. + base := "https://www.flightradar24.com/data/airports/brazil" + + cases := map[string]string{ + "/data/airports/brazil/sp": "https://www.flightradar24.com/data/airports/brazil/sp", + "https://www.flightradar24.com/data/airports/x/y": "https://www.flightradar24.com/data/airports/x/y", + "rj": "https://www.flightradar24.com/data/airports/rj", + "::not a url": "::not a url", + } + + for reference, want := range cases { + if got := resolveReference(base, reference); got != want { + t.Errorf("%q: got %q, want %q", reference, got, want) + } + } + if got := resolveReference("::broken base", "/data/airports/brazil/sp"); got != "/data/airports/brazil/sp" { + t.Errorf("got %q, want the path as given", got) } } diff --git a/go/flightradarapi/core.go b/go/flightradarapi/core.go index 08ba314..209f338 100644 --- a/go/flightradarapi/core.go +++ b/go/flightradarapi/core.go @@ -30,7 +30,8 @@ type endpoints struct { apiAirportData string airportData string - airportsJSON string + airportsData string + airportsByCountry string airportDisruption string airlinesData string @@ -56,9 +57,15 @@ func defaultEndpoints() endpoints { historicalData: flightRadarBaseURL + "/download/?flight=%s&file=%s&trailLimit=0&history=%d", - apiAirportData: apiFlightRadarBaseURL + "/airport.json", - airportData: flightRadarBaseURL + "/airports/traffic-stats/?airport=%s", - airportsJSON: flightRadarBaseURL + "/_json/airports.php", + apiAirportData: apiFlightRadarBaseURL + "/airport.json", + airportData: flightRadarBaseURL + "/airports/traffic-stats/?airport=%s", + // The listing is split by country, and by state for the largest + // countries. These are Inertia pages: the data rides JSON-encoded in the + // data-page attribute of the HTML, or comes as the whole body when the + // request carries inertiaHeaders and the asset version the index page + // declares. + airportsData: flightRadarBaseURL + "/data/airports", + airportsByCountry: flightRadarBaseURL + "/data/airports/%s", airportDisruption: flightRadarBaseURL + "/webapi/v1/airport-disruptions", airlinesData: flightRadarBaseURL + "/data/airlines", @@ -97,6 +104,10 @@ func (e endpoints) countryFlagURL(slug string) string { return fmt.Sprintf(e.countryFlag, url.PathEscape(slug)) } +func (e endpoints) airportsByCountryURL(slug string) string { + return fmt.Sprintf(e.airportsByCountry, url.PathEscape(slug)) +} + func (e endpoints) airlineLogoURL(iata, icao string) string { return fmt.Sprintf(e.airlineLogo, url.PathEscape(iata), url.PathEscape(icao)) } @@ -130,6 +141,13 @@ var ( imageHeaders = withHeaders(baseHeaders, map[string]string{ "accept": "image/gif, image/jpg, image/jpeg, image/png", }) + // inertiaHeaders ask an Inertia page for its page object instead of the + // HTML around it. The request must also carry an x-inertia-version matching + // the deployed assets, or FR24 answers 409. + inertiaHeaders = withHeaders(jsonHeaders, map[string]string{ + "x-inertia": "true", + "x-requested-with": "XMLHttpRequest", + }) htmlHeaders = map[string]string{ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8," + diff --git a/go/flightradarapi/parsers.go b/go/flightradarapi/parsers.go index 57fd6b8..83f2f9c 100644 --- a/go/flightradarapi/parsers.go +++ b/go/flightradarapi/parsers.go @@ -4,11 +4,9 @@ import ( "bytes" "encoding/json" "errors" - "io" "math" "reflect" "regexp" - "slices" "strconv" "strings" "unicode" @@ -25,6 +23,16 @@ var numericPattern = regexp.MustCompile(`^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+ // U+001C-U+001F and U+FEFF. const surroundingSpace = " \t\n\r\f\v" +// airlineCodePattern is an airline code badge: a two-letter IATA code or a +// three-letter ICAO code. ASCII only, so a stray Unicode letter does not pass +// for a code. +var airlineCodePattern = regexp.MustCompile(`^[A-Z0-9]{2,3}$`) + +// aircraftCountPattern is a fleet size as the page prints it: "340" or "3,878". +var aircraftCountPattern = regexp.MustCompile(`^[0-9]{1,3}(?:[,.][0-9]{3})*$|^[0-9]+$`) + +const airlinesPathPrefix = "/data/airlines/" + // Airline is one row of the airlines listing. The JSON tags match the keys the // Python and Node.js ports use, so NumAircrafts serialises as "n_aircrafts". type Airline struct { @@ -35,6 +43,11 @@ type Airline struct { } // parseAirlinesHTML parses the airlines listing page into airline records. +// +// Every airline is a link to its data page, with the IATA and ICAO codes as +// badges next to the name. The "Largest fleets" section repeats a few airlines +// with their aircraft count; an airline the page lists twice becomes one entry +// carrying everything the page said about it. func parseAirlinesHTML(page []byte) []Airline { document, err := html.Parse(bytes.NewReader(page)) @@ -43,71 +56,114 @@ func parseAirlinesHTML(page []byte) []Airline { return []Airline{} } - tbody := findElement(document, func(node *html.Node) bool { return node.Data == "tbody" }) + links := findElements(document, func(node *html.Node) bool { + return node.Data == "a" && strings.HasPrefix(attr(node, "href"), airlinesPathPrefix) + }) - if tbody == nil { - log().Warn("parseAirlinesHTML: no in response — FR24 page layout may have changed") + if len(links) == 0 { + log().Warn("parseAirlinesHTML: no airline links in response — FR24 page layout may have changed") return []Airline{} } airlines := []Airline{} + indexByHref := map[string]int{} - for _, row := range findElements(tbody, func(node *html.Node) bool { return node.Data == "tr" }) { - cells := findElements(row, func(node *html.Node) bool { return node.Data == "td" }) - notranslate := findElement(row, func(node *html.Node) bool { - return node.Data == "td" && hasClass(node, "notranslate") - }) + for _, link := range links { + name := airlineName(link) - if notranslate == nil { + if len(name) < 2 { continue } - link := findElement(notranslate, func(node *html.Node) bool { - return node.Data == "a" && strings.HasPrefix(attr(node, "href"), "/data/airlines") - }) + var iata, icao string - if link == nil { - continue + for _, text := range leafTexts(link, "span") { + switch { + case !airlineCodePattern.MatchString(text): + case len(text) == 2 && iata == "": + iata = text + case len(text) == 3 && icao == "": + icao = text + } } - name := textContent(link) - - if len(name) < 2 { + // Section links ("All") point under /data/airlines/ too; a code tells + // an airline apart from them. + if iata == "" && icao == "" { continue } - airline := Airline{Name: name} + var numAircrafts *int - if len(cells) >= 4 { - codes := textContent(cells[3]) + for _, text := range leafTexts(link, "strong") { + if aircraftCountPattern.MatchString(text) { + digits := strings.Map(func(char rune) rune { + if char >= '0' && char <= '9' { + return char + } + return -1 + }, text) - switch { - case strings.Contains(codes, " / "): - if parts := strings.Split(codes, " / "); len(parts) == 2 { - airline.IATA = strings.TrimSpace(parts[0]) - airline.ICAO = strings.TrimSpace(parts[1]) + if count, err := strconv.Atoi(digits); err == nil { + numAircrafts = &count + break } - case len(codes) == 2: - airline.IATA = codes - case len(codes) == 3: - airline.ICAO = codes } } - if len(cells) >= 5 { - if text := textContent(cells[4]); text != "" { - field := strings.TrimSpace(strings.SplitN(text, " ", 2)[0]) + href := attr(link, "href") + index, seen := indexByHref[href] - if count, err := strconv.Atoi(field); err == nil { - airline.NumAircrafts = &count - } - } + if !seen { + index = len(airlines) + indexByHref[href] = index + airlines = append(airlines, Airline{Name: name}) + } + + airline := &airlines[index] + + if airline.ICAO == "" { + airline.ICAO = icao + } + if airline.IATA == "" { + airline.IATA = iata + } + if airline.NumAircrafts == nil { + airline.NumAircrafts = numAircrafts } - airlines = append(airlines, airline) } return airlines } +// airlineName is the airline name of a listing entry. The name sits in the +// element titled with it, with the logo's aria-label as a fallback; the anchor +// text also carries the codes and the fleet size. +func airlineName(link *html.Node) string { + for _, attribute := range []string{"title", "aria-label"} { + element := findElement(link, func(node *html.Node) bool { return hasAttr(node, attribute) }) + + if element != nil { + return strings.TrimSpace(attr(element, attribute)) + } + } + return textContent(link) +} + +// leafTexts is the text of the `tag` elements under root that hold no other +// element. The name element is left out by its title attribute: an airline +// called "TAP" would otherwise read as an ICAO code. +func leafTexts(root *html.Node, tag string) []string { + var texts []string + + for _, element := range findElements(root, func(node *html.Node) bool { return node.Data == tag }) { + if hasAttr(element, "title") || findElement(element, func(*html.Node) bool { return true }) != nil { + continue + } + texts = append(texts, textContent(element)) + } + return texts +} + // countryToSlug slugifies a country name the way FR24 spells it in its data page // URLs, so feed rows can be matched against the Country constants. func countryToSlug(country string) string { @@ -214,49 +270,79 @@ func parseNumber(text string) *float64 { return &number } -// parseAirportsJSON parses the airports JSON feed into airports, keeping only -// the wanted countries when any are given. -func parseAirportsJSON(payload []byte, countries []Country) []*Airport { - decoder := json.NewDecoder(bytes.NewReader(bytes.TrimPrefix(payload, []byte("\xef\xbb\xbf")))) - - // Numbers stay as text, so a literal no float64 can hold is rejected - // instead of silently becoming an infinity. - decoder.UseNumber() +// parseInertiaPage returns the Inertia page object (component, props, version) +// of a FR24 data page, or false when the payload carries none. +// +// Served as HTML, the page keeps it JSON-encoded in the data-page attribute of +// its root element; asked with inertiaHeaders, FR24 sends it as the body, which +// the caller has already decoded into a map. Both shapes are accepted, so a +// test can feed the parser a decoded fixture the way the other ports do. +func parseInertiaPage(payload any) (map[string]any, bool) { + var page any + + switch typed := payload.(type) { + case map[string]any: + page = typed + case string: + return parseInertiaPage([]byte(typed)) + case []byte: + document, err := html.Parse(bytes.NewReader(typed)) - var data map[string]any + if err != nil { + log().Warn("parseInertiaPage: could not parse the response as HTML") + return nil, false + } - if err := decoder.Decode(&data); err != nil { - log().Warn("parseAirportsJSON: response is not valid JSON — FR24 feed may have changed") - return []*Airport{} - } + root := findElement(document, func(node *html.Node) bool { return hasAttr(node, "data-page") }) - // Decode stops at the first value, so a truncated or spliced body would - // otherwise pass as valid. The Python and Node.js parsers reject it. - if _, err := decoder.Token(); !errors.Is(err, io.EOF) { - log().Warn("parseAirportsJSON: response carries data past the JSON body — FR24 feed may have changed") - return []*Airport{} + if root == nil { + log().Warn("parseInertiaPage: no data-page attribute in response — FR24 page layout may have changed") + return nil, false + } + if err := json.Unmarshal([]byte(attr(root, "data-page")), &page); err != nil { + log().Warn("parseInertiaPage: data-page is not valid JSON — FR24 page layout may have changed") + return nil, false + } + default: + page = payload } - rows, ok := data["rows"].([]any) + object, ok := page.(map[string]any) if !ok { - log().Warn(`parseAirportsJSON: no "rows" array in response — FR24 feed may have changed`) - return []*Airport{} + log().Warn("parseInertiaPage: no page props in response — FR24 page layout may have changed") + return nil, false + } + if _, ok := object["props"].(map[string]any); !ok { + log().Warn("parseInertiaPage: no page props in response — FR24 page layout may have changed") + return nil, false } + return object, true +} - var wanted map[string]bool +// airportsCountry is one entry of the airports index: the country name as FR24 +// prints it ("UNITED STATES") and the slug of its listing page, which is also +// its Country constant. +type airportsCountry struct { + Name string + Slug string +} - if countries != nil { - wanted = make(map[string]bool, len(countries)) +// parseAirportsIndex parses the airports index (endpoints.airportsData) into +// its countries. The payload is the page HTML, or its decoded page object. +func parseAirportsIndex(payload any) []airportsCountry { + page, ok := parseInertiaPage(payload) + var rows []any - for _, country := range countries { - wanted[countryToSlug(string(country))] = true - } + if ok { + rows, _ = page["props"].(map[string]any)["airportsByCountry"].([]any) + } + if rows == nil { + log().Warn(`parseAirportsIndex: no "airportsByCountry" list in response — FR24 page layout may have changed`) + return []airportsCountry{} } - matched := make(map[string]bool, len(wanted)) - var unpositioned []string - airports := []*Airport{} + countries := []airportsCountry{} for _, entry := range rows { row, ok := entry.(map[string]any) @@ -265,58 +351,63 @@ func parseAirportsJSON(payload []byte, countries []Country) []*Airport { continue } - // Slugified only when filtering: this loop runs over every airport in - // the feed. - if wanted != nil { - slug := countryToSlug(toText(row["country"])) + name := toText(row["name"]) + slug := countryToSlug(name) - if !wanted[slug] { - continue - } - matched[slug] = true + if slug != "" { + countries = append(countries, airportsCountry{Name: name, Slug: slug}) } + } + return countries +} - latitude := toNumber(row["lat"]) - longitude := toNumber(row["lon"]) +// parseAirportsPage parses a country or state airports page into its airports +// and the paths of its state pages. The payload is the page HTML, or its +// decoded page object. +// +// A country with many airports lists its states instead, each with the path of +// its own page; those paths come back for the caller to fetch. The listing has +// no coordinates or altitude: [Client.GetAirport] has them for one airport. +func parseAirportsPage(payload any) (airports []*Airport, statePaths []string) { + airports, statePaths = []*Airport{}, []string{} + page, ok := parseInertiaPage(payload) - // One bad coordinate drops both: half a position reads as located. - if latitude == nil || longitude == nil { - latitude, longitude = nil, nil - unpositioned = append(unpositioned, toText(row["name"])) - } + if !ok { + return airports, statePaths + } + + props := page["props"].(map[string]any) + country, _ := props["country"].(map[string]any) + countryName := toText(country["name"]) + + rows, _ := props["airports"].([]any) + states, _ := props["states"].([]any) + for _, entry := range rows { + row, ok := entry.(map[string]any) + + if !ok { + continue + } airports = append(airports, newAirportFromBasicInfo(basicAirportInfo{ - Name: toText(row["name"]), - ICAO: toText(row["icao"]), - IATA: toText(row["iata"]), - Latitude: latitude, - Longitude: longitude, - Altitude: toNumber(row["alt"]), - Country: toText(row["country"]), + Name: toText(row["name"]), + ICAO: toText(row["icao"]), + IATA: toText(row["iata"]), + Country: countryName, })) } - // One line, not one per row: the feed carries every airport. - if len(unpositioned) > 0 { - log().Warn("parseAirportsJSON: airports with unusable coordinates carry no position", - "count", len(unpositioned), "examples", unpositioned[:min(3, len(unpositioned))]) - } - - if wanted != nil { - var missing []string + for _, entry := range states { + state, ok := entry.(map[string]any) - for slug := range wanted { - if !matched[slug] { - missing = append(missing, slug) - } + if !ok { + continue } - if len(missing) > 0 { - slices.Sort(missing) - log().Warn("parseAirportsJSON: no airports found for some countries — check the Country constants", - "countries", strings.Join(missing, ", ")) + if path := toText(state["url"]); path != "" { + statePaths = append(statePaths, path) } } - return airports + return airports, statePaths } // --- HTML helpers --- @@ -350,8 +441,15 @@ func attr(node *html.Node, name string) string { return "" } -func hasClass(node *html.Node, name string) bool { - return slices.Contains(strings.Fields(attr(node, "class")), name) +// hasAttr reports whether the element carries the attribute at all, which +// attr cannot tell from an empty value. +func hasAttr(node *html.Node, name string) bool { + for _, attribute := range node.Attr { + if attribute.Key == name { + return true + } + } + return false } // textContent concatenates the node's text, trimming each piece, as diff --git a/go/flightradarapi/parsers_test.go b/go/flightradarapi/parsers_test.go index 2bd0718..219dc3b 100644 --- a/go/flightradarapi/parsers_test.go +++ b/go/flightradarapi/parsers_test.go @@ -2,10 +2,9 @@ package flightradarapi import ( "encoding/json" - "fmt" - "math" "os" "path/filepath" + "slices" "strings" "testing" ) @@ -35,28 +34,6 @@ func airlinesByName(t *testing.T) map[string]Airline { return byName } -func airportsByIATA(t *testing.T, airports []*Airport) map[string]*Airport { - t.Helper() - byIATA := map[string]*Airport{} - - for _, airport := range airports { - byIATA[airport.IATA] = airport - } - return byIATA -} - -// airportsFromRow parses a one-row feed built from a raw JSON literal, the way -// both other ports do it. -func airportsFromRow(t *testing.T, row string) []*Airport { - t.Helper() - airports := parseAirportsJSON([]byte(`{"rows":[`+row+`]}`), nil) - - if len(airports) != 1 { - t.Fatalf("got %d airports for row %s, want 1", len(airports), row) - } - return airports -} - // --- parseAirlinesHTML --- func TestParseAirlinesHTMLExtractsKnownRows(t *testing.T) { @@ -102,289 +79,327 @@ func TestParseAirlinesHTMLParsesAircraftCount(t *testing.T) { for _, expected := range []struct { name string count int - }{{"LATAM Airlines", 340}, {"Gol", 140}} { + }{ + {"LATAM Airlines", 340}, + {"Gol", 140}, + // Thousands separator, as the page prints large fleets. + {"Delta Air Lines", 1900}, + } { count := byName[expected.name].NumAircrafts if count == nil || *count != expected.count { t.Errorf("%s: got %v aircraft, want %d", expected.name, count, expected.count) } } + + // Only the "Largest fleets" section carries a count. + if count := byName["Sky2"].NumAircrafts; count != nil { + t.Errorf("Sky2: got %d aircraft, want none", *count) + } +} + +func TestParseAirlinesHTMLMergesAnAirlineListedInTwoSections(t *testing.T) { + // Delta and Gol appear under "Largest fleets" and again under "All airlines". + seen := map[string]int{} + + for _, airline := range parseAirlinesHTML(loadFixture(t, "airlines.html")) { + seen[airline.Name]++ + } + for _, name := range []string{"Delta Air Lines", "Gol"} { + if seen[name] != 1 { + t.Errorf("%s: listed %d times, want once", name, seen[name]) + } + } +} + +func TestParseAirlinesHTMLKeepsACodeShapedName(t *testing.T) { + tap := airlinesByName(t)["TAP"] + + if tap.Name != "TAP" || tap.ICAO != "TAP" || tap.IATA != "TP" || tap.NumAircrafts != nil { + t.Errorf("got %+v, want TAP / TAP / TP with no count", tap) + } } func TestParseAirlinesHTMLSkipsInvalidRows(t *testing.T) { airlines := parseAirlinesHTML(loadFixture(t, "airlines.html")) - // 5 valid rows; 2 invalid (no notranslate, wrong href) must be skipped. - if len(airlines) != 5 { - t.Errorf("got %d airlines, want 5", len(airlines)) + // 6 airlines; the section link, the entry with no link, the wrong href and + // the one-letter name must be skipped. + if len(airlines) != 6 { + t.Errorf("got %d airlines, want 6", len(airlines)) + } + for _, airline := range airlines { + if airline.Name == "All" { + t.Errorf("the section link was read as an airline: %+v", airline) + } + } +} + +func TestParseAirlinesHTMLKeepsTheOrderOfThePage(t *testing.T) { + var names []string + + for _, airline := range parseAirlinesHTML(loadFixture(t, "airlines.html")) { + names = append(names, airline.Name) + } + + want := []string{"Delta Air Lines", "LATAM Airlines", "Gol", "Sky2", "SkyTeam", "TAP"} + + if !slices.Equal(names, want) { + t.Errorf("got %v, want %v", names, want) } } func TestParseAirlinesHTMLEmptyInputReturnsEmptyList(t *testing.T) { - for _, page := range []string{"", "

no tbody here

"} { + for _, page := range []string{"", "

no airlines here

"} { if airlines := parseAirlinesHTML([]byte(page)); len(airlines) != 0 { t.Errorf("got %d airlines for %q, want 0", len(airlines), page) } } } -// --- parseAirportsJSON --- +// --- parseInertiaPage --- -func TestParseAirportsJSONExtractsBasicFields(t *testing.T) { - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) +func decodeFixture(t *testing.T, name string) map[string]any { + t.Helper() + var page map[string]any - for _, code := range []string{"GRU", "GIG"} { - if _, ok := byIATA[code]; !ok { - t.Fatalf("missing airport %q", code) - } + if err := json.Unmarshal(loadFixture(t, name), &page); err != nil { + t.Fatalf("could not decode fixture %s: %v", name, err) } + return page +} - gru := byIATA["GRU"] +func TestParseInertiaPageReadsTheDataPageAttribute(t *testing.T) { + page, ok := parseInertiaPage(loadFixture(t, "airports_index.html")) - if gru.ICAO != "SBGR" || gru.Country != "Brazil" { - t.Errorf("GRU: got ICAO %q country %q", gru.ICAO, gru.Country) - } - if gru.Latitude == nil || math.Abs(*gru.Latitude-(-23.429991)) > 1e-6 { - t.Errorf("GRU latitude: got %v", gru.Latitude) + if !ok { + t.Fatal("no page object found") } - if gru.Longitude == nil || math.Abs(*gru.Longitude-(-46.4674)) > 1e-6 { - t.Errorf("GRU longitude: got %v", gru.Longitude) + if page["component"] != "Data/Airports" || page["version"] != "919b94d2a33f365a34f99828f6eeec6a" { + t.Errorf("got component %v version %v", page["component"], page["version"]) } - if gru.Altitude == nil || *gru.Altitude != 2436 { - t.Errorf("GRU altitude: got %v, want 2436", gru.Altitude) + if _, ok := page["props"].(map[string]any)["airportsByCountry"]; !ok { + t.Errorf("got props %v, want airportsByCountry among them", page["props"]) } } -func TestParseAirportsJSONFormatsAltitudeWithoutDecimals(t *testing.T) { - // Both other ports report 2436, never 2436.0. - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) +func TestParseInertiaPageAcceptsADecodedJSONBody(t *testing.T) { + body := decodeFixture(t, "airports_country.json") + page, ok := parseInertiaPage(body) - if got := formatNumber(*byIATA["GRU"].Altitude); got != "2436" { - t.Errorf("got altitude %q, want \"2436\"", got) + if !ok || page["component"] != body["component"] { + t.Errorf("got %v, want the body handed back", page) } } -func TestParseAirportsJSONKeepsEveryCountryWithoutFilter(t *testing.T) { - countries := map[string]bool{} - - for _, airport := range parseAirportsJSON(loadFixture(t, "airports.json"), nil) { - countries[airport.Country] = true +func TestParseInertiaPageAcceptsAString(t *testing.T) { + if _, ok := parseInertiaPage(string(loadFixture(t, "airports_index.html"))); !ok { + t.Error("the HTML as a string was not read") } +} - for _, country := range []string{"Brazil", "United States", "Spain"} { - if !countries[country] { - t.Errorf("missing country %q", country) +func TestParseInertiaPageRejectsUnrecognisedPayloads(t *testing.T) { + for _, payload := range []any{ + []byte(""), + []byte(`
`), + []byte(`
`), + []byte(`
`), + map[string]any{"component": "Data/Airports"}, + map[string]any{"props": "not an object"}, + []any{1, 2}, + nil, + 42, + } { + if page, ok := parseInertiaPage(payload); ok || page != nil { + t.Errorf("%v: got %v, want nothing", payload, page) } } } -func TestParseAirportsJSONFiltersByCountrySlug(t *testing.T) { - airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{"united-states"}) +// --- parseAirportsIndex --- + +func TestParseAirportsIndexListsEveryCountryWithItsSlug(t *testing.T) { + countries := parseAirportsIndex(loadFixture(t, "airports_index.html")) + var slugs []string - if len(airports) == 0 { - t.Fatal("no airports returned") + for _, country := range countries { + slugs = append(slugs, country.Slug) } - for _, airport := range airports { - if airport.Country != "United States" { - t.Errorf("got country %q, want United States", airport.Country) - } + if want := []string{"brazil", "myanmar-burma", "uruguay"}; !slices.Equal(slugs, want) { + t.Fatalf("got %v, want %v", slugs, want) + } + if countries[1].Name != "MYANMAR (BURMA)" { + t.Errorf("got name %q, want it as FR24 prints it", countries[1].Name) } } -func TestParseAirportsJSONAcceptsSeveralCountries(t *testing.T) { - airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{"brazil", "spain"}) - countries := map[string]bool{} +func TestParseAirportsIndexSlugsMatchTheCountryConstants(t *testing.T) { + known := map[Country]bool{} - for _, airport := range airports { - countries[airport.Country] = true + for _, country := range AllCountries() { + known[country] = true } - if len(countries) != 2 || !countries["Brazil"] || !countries["Spain"] { - t.Errorf("got countries %v, want Brazil and Spain", countries) + for _, country := range parseAirportsIndex(loadFixture(t, "airports_index.html")) { + if !known[Country(country.Slug)] { + t.Errorf("%q is not a Country constant", country.Slug) + } } } -func TestParseAirportsJSONInvalidCoordinatesBecomeNil(t *testing.T) { - // Regression: invalid coords used to be coerced to (0, 0), placing the - // airport in the Gulf of Guinea. - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) - bad := byIATA["BAD"] - - if bad == nil { - t.Fatal("missing airport BAD") - } - if bad.Latitude != nil || bad.Longitude != nil { - t.Errorf("got position (%v, %v), want no position", bad.Latitude, bad.Longitude) +func TestParseAirportsIndexInvalidPayloadReturnsEmptyList(t *testing.T) { + for _, payload := range []any{ + []byte(""), + map[string]any{"props": map[string]any{}}, + map[string]any{"props": map[string]any{"airportsByCountry": "x"}}, + map[string]any{"props": map[string]any{"airportsByCountry": []any{1, "x", map[string]any{"name": nil}}}}, + } { + if countries := parseAirportsIndex(payload); len(countries) != 0 { + t.Errorf("%v: got %v, want none", payload, countries) + } } } -func TestParseAirportsJSONRejectsNumericLookingJunk(t *testing.T) { - payload, err := json.Marshal(map[string]any{"rows": []any{map[string]any{ - "name": "Junk Airport", "iata": "JNK", "icao": "JJNK", - "lat": "43.30 N", "lon": -8.37725, "country": "Spain", "alt": "-1", - }}}) - - if err != nil { - t.Fatal(err) - } +// --- parseAirportsPage --- - airport := parseAirportsJSON(payload, nil)[0] +func iatas(airports []*Airport) []string { + codes := make([]string, 0, len(airports)) - // One bad coordinate drops both; altitude is independent and survives. - if airport.Latitude != nil || airport.Longitude != nil { - t.Errorf("got position (%v, %v), want no position", airport.Latitude, airport.Longitude) - } - if airport.Altitude == nil || *airport.Altitude != -1 { - t.Errorf("got altitude %v, want -1", airport.Altitude) + for _, airport := range airports { + codes = append(codes, airport.IATA) } + return codes } -func TestParseAirportsJSONAgreesWithOtherPortsOnUnicode(t *testing.T) { - cases := []struct { - name string - value string - expected *float64 - }{ - // Arabic-Indic digits, which a Unicode-aware \d matched before. - {"arabic-indic digits", "٤٣", nil}, - // U+FEFF, which JavaScript's trim() drops and Python's strip() keeps. - {"leading byte-order mark", "\ufeff43", nil}, - // U+001C, which Python's strip() drops and JavaScript's trim() keeps. - {"leading file separator", "\u001c43", nil}, - {"plain spaces are trimmed", " 43 ", float64Ptr(43)}, +func TestParseAirportsPageExtractsTheAirportsOfACountry(t *testing.T) { + airports, statePaths := parseAirportsPage(decodeFixture(t, "airports_country.json")) + + if len(statePaths) != 0 { + t.Errorf("got state paths %v, want none", statePaths) + } + if got := iatas(airports); !slices.Equal(got, []string{"CYR", "MVD"}) { + t.Fatalf("got %v, want CYR and MVD", got) } - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - row, err := json.Marshal(map[string]any{ - "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", - "lat": testCase.value, "lon": testCase.value, "alt": testCase.value, - }) + mvd := airports[1] - if err != nil { - t.Fatal(err) - } + if mvd.Name != "Montevideo Carrasco International Airport" || mvd.ICAO != "SUMU" || mvd.Country != "Uruguay" { + t.Errorf("got %q / %q / %q", mvd.Name, mvd.ICAO, mvd.Country) + } +} - airport := airportsFromRow(t, string(row))[0] +func TestParseAirportsPageListingCarriesNoPosition(t *testing.T) { + airports, _ := parseAirportsPage(decodeFixture(t, "airports_country.json")) - if !equalOptional(airport.Altitude, testCase.expected) { - t.Errorf("got altitude %v, want %v", airport.Altitude, testCase.expected) - } - }) + for _, airport := range airports { + if airport.Latitude != nil || airport.Longitude != nil || airport.Altitude != nil { + t.Errorf("%s: got position (%v, %v) altitude %v, want none", + airport.IATA, airport.Latitude, airport.Longitude, airport.Altitude) + } } } -func TestParseAirportsJSONNumericCoercion(t *testing.T) { - // Keep in step with the same list in the Python and Node.js suites. - cases := []struct { - name string - literal string - expected *float64 - }{ - {"whitespace", `" "`, nil}, - {"an array holding a number", `[43]`, nil}, - {"a coordinate with a hemisphere suffix", `"43.30 N"`, nil}, - {"an exponent overflowing a double", `"1e999"`, nil}, - {"400 plain digits", `"` + repeat("1", 400) + `"`, nil}, - {"a whole number as a string", `"2436"`, float64Ptr(2436)}, - {"a negative decimal", `-23.4`, float64Ptr(-23.4)}, - {"a genuine zero", `0`, float64Ptr(0)}, - } - - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - row := fmt.Sprintf( - `{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":%s,"lon":%s,"alt":%s}`, - testCase.literal, testCase.literal, testCase.literal) - airport := airportsFromRow(t, row)[0] - - for label, got := range map[string]*float64{ - "latitude": airport.Latitude, - "longitude": airport.Longitude, - "altitude": airport.Altitude, - } { - if !equalOptional(got, testCase.expected) { - t.Errorf("%s: got %v, want %v", label, got, testCase.expected) - } - } - }) - } -} - -func TestParseAirportsJSONSurvivesRawNumbersNoDoubleCanHold(t *testing.T) { - for _, literal := range []string{repeat("1", 400), "9007199254740993", "-" + repeat("1", 400)} { - row := fmt.Sprintf( - `{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":%s,"lon":1,"alt":2}`, literal) - latitude := airportsFromRow(t, row)[0].Latitude - - if latitude != nil && (math.IsInf(*latitude, 0) || math.IsNaN(*latitude)) { - t.Errorf("%s: got %v, want nil or a finite number", literal, *latitude) - } +func TestParseAirportsPageReturnsTheStatePagesOfASplitCountry(t *testing.T) { + airports, statePaths := parseAirportsPage(decodeFixture(t, "airports_country_states.json")) + + if len(airports) != 0 { + t.Errorf("got %d airports, want none", len(airports)) + } + if want := []string{"/data/airports/brazil/rj", "/data/airports/brazil/sp"}; !slices.Equal(statePaths, want) { + t.Errorf("got %v, want %v", statePaths, want) } } -func TestParseAirportsJSONKeepsTextFieldsAsStrings(t *testing.T) { - // Anything else breaks GetCountryFlag(airport.Country). - for _, literal := range []string{"null", "0", "false", "true", "[]", "{}", "123"} { - row := fmt.Sprintf( - `{"name":%s,"iata":%s,"icao":%s,"country":%s,"lat":1,"lon":2,"alt":3}`, - literal, literal, literal, literal) - airport := airportsFromRow(t, row)[0] - - if airport.Name != "" || airport.IATA != "" || airport.ICAO != "" || airport.Country != "" { - t.Errorf("%s: got %q/%q/%q/%q, want empty strings", - literal, airport.Name, airport.IATA, airport.ICAO, airport.Country) +func TestParseAirportsPageStatePageKeepsTheCountryName(t *testing.T) { + airports, statePaths := parseAirportsPage(decodeFixture(t, "airports_state_sp.json")) + + if len(statePaths) != 0 { + t.Errorf("got state paths %v, want none", statePaths) + } + if got := iatas(airports); !slices.Equal(got, []string{"GRU", "CGH"}) { + t.Errorf("got %v, want GRU and CGH", got) + } + for _, airport := range airports { + if airport.Country != "Brazil" { + t.Errorf("%s: got country %q, want Brazil", airport.IATA, airport.Country) } } } -func TestParseAirportsJSONCountrySpellingAndSlugRoundTrip(t *testing.T) { - byIATA := airportsByIATA(t, parseAirportsJSON(loadFixture(t, "airports.json"), nil)) - ann := byIATA["VBA"] +func TestParseAirportsPageReadsAnHTMLPage(t *testing.T) { + airports, _ := parseAirportsPage(loadFixture(t, "airports_country.html")) - if ann == nil { - t.Fatal("missing airport VBA") + if got := iatas(airports); !slices.Equal(got, []string{"RGN"}) { + t.Fatalf("got %v, want RGN", got) } - if ann.Country != "Myanmar (Burma)" { - t.Errorf("got country %q", ann.Country) + // FR24 spells it "Myanmar (Burma)" on the page and "myanmar-burma" in URLs. + if airports[0].Country != "Myanmar (Burma)" { + t.Errorf("got country %q", airports[0].Country) } - if slug := countryToSlug(ann.Country); slug != "myanmar-burma" { - t.Errorf("got slug %q, want myanmar-burma", slug) - } - if ann.Altitude == nil || *ann.Altitude != 43 { - t.Errorf("got altitude %v, want 43", ann.Altitude) + if slug := countryToSlug(airports[0].Country); slug != string(CountryMyanmarBurma) { + t.Errorf("got slug %q, want %q", slug, CountryMyanmarBurma) } } -func TestParseAirportsJSONFilterAcceptsEitherCountrySpelling(t *testing.T) { - for _, spelling := range []Country{"myanmar-burma", "Myanmar (Burma)", CountryMyanmarBurma} { - airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{spelling}) +func TestParseAirportsPageSkipsRowsThatAreNotObjects(t *testing.T) { + payload := map[string]any{"props": map[string]any{ + "country": map[string]any{"name": "Nowhere"}, + "airports": []any{1, "x", nil, map[string]any{"iata": "RRR"}}, + }} + airports, _ := parseAirportsPage(payload) - if len(airports) != 1 { - t.Errorf("%q: got %d airports, want 1", spelling, len(airports)) - } + if got := iatas(airports); !slices.Equal(got, []string{"RRR"}) { + t.Errorf("got %v, want only RRR", got) } } -func TestParseAirportsJSONSkipsRowsThatAreNotObjects(t *testing.T) { - payload := []byte(`{"rows":[[1,2,3],"x",7,null,` + - `{"name":"Real","iata":"RRR","icao":"RRRR","country":"Spain","lat":1,"lon":2,"alt":3}]}`) - airports := parseAirportsJSON(payload, nil) +func TestParseAirportsPageKeepsTextFieldsAsStrings(t *testing.T) { + // Anything else breaks GetCountryFlag(airport.Country). + payload := map[string]any{"props": map[string]any{ + "country": map[string]any{"name": []any{"Brazil"}}, + "airports": []any{map[string]any{"name": 42.0, "iata": nil, "icao": map[string]any{"x": 1.0}}}, + }} + airports, _ := parseAirportsPage(payload) + + if len(airports) != 1 { + t.Fatalf("got %d airports, want 1", len(airports)) + } + + airport := airports[0] - if len(airports) != 1 || airports[0].IATA != "RRR" { - t.Errorf("got %d airports, want only RRR", len(airports)) + if airport.Name != "" || airport.IATA != "" || airport.ICAO != "" || airport.Country != "" { + t.Errorf("got %q/%q/%q/%q, want empty strings", airport.Name, airport.IATA, airport.ICAO, airport.Country) } } -func TestParseAirportsJSONUnknownCountryReturnsEmptyList(t *testing.T) { - if airports := parseAirportsJSON(loadFixture(t, "airports.json"), []Country{"atlantis"}); len(airports) != 0 { - t.Errorf("got %d airports, want 0", len(airports)) +func TestParseAirportsPageIgnoresMalformedStates(t *testing.T) { + payload := map[string]any{"props": map[string]any{"states": []any{ + map[string]any{"url": "/data/airports/x/a"}, + map[string]any{"url": ""}, + map[string]any{"code": "B"}, + "junk", + map[string]any{"url": 3.0}, + }}} + airports, statePaths := parseAirportsPage(payload) + + if len(airports) != 0 || !slices.Equal(statePaths, []string{"/data/airports/x/a"}) { + t.Errorf("got %d airports and state paths %v, want only /data/airports/x/a", len(airports), statePaths) } } -func TestParseAirportsJSONInvalidPayloadReturnsEmptyList(t *testing.T) { - for _, payload := range []string{"", "{}", "not json"} { - if airports := parseAirportsJSON([]byte(payload), nil); len(airports) != 0 { - t.Errorf("%q: got %d airports, want 0", payload, len(airports)) +func TestParseAirportsPageInvalidPayloadReturnsNothing(t *testing.T) { + for _, payload := range []any{ + []byte(""), + []byte("not a data page"), + map[string]any{"props": map[string]any{}}, + map[string]any{"props": map[string]any{"country": "Brazil", "airports": "none", "states": 7.0}}, + } { + airports, statePaths := parseAirportsPage(payload) + + if airports == nil || statePaths == nil { + t.Errorf("%v: got nil slices, want empty ones", payload) + } + if len(airports) != 0 || len(statePaths) != 0 { + t.Errorf("%v: got %d airports and %v, want nothing", payload, len(airports), statePaths) } } } @@ -431,46 +446,12 @@ func TestCountryConstantsRoundTripThroughTheSlugifier(t *testing.T) { } } -func equalOptional(got, want *float64) bool { - if got == nil || want == nil { - return got == nil && want == nil - } - return *got == *want -} - // float64Ptr is the shorthand the table-driven tests need for optional numbers. func float64Ptr(value float64) *float64 { return &value } -func repeat(text string, times int) string { - result := make([]byte, 0, len(text)*times) - - for range times { - result = append(result, text...) - } - return string(result) -} - -func TestParseAirportsJSONRejectsDataPastTheJSONBody(t *testing.T) { - // Decode reads the first value only, so a spliced body used to pass. - payload := []byte(`{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",` + - `"lat":1,"lon":2,"alt":3}]}trailing garbage`) - - if airports := parseAirportsJSON(payload, nil); len(airports) != 0 { - t.Errorf("got %d airports, want the body rejected", len(airports)) - } - - // Trailing whitespace is not garbage. - clean := []byte(`{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",` + - `"lat":1,"lon":2,"alt":3}]}` + "\n\n ") - - if airports := parseAirportsJSON(clean, nil); len(airports) != 1 { - t.Errorf("got %d airports, want the body accepted", len(airports)) - } -} - func TestTextFieldsReadAPayloadDecodedWithUseNumber(t *testing.T) { - // This package decodes the airports feed that way, and the exported - // constructors take a map the caller may have decoded the same way. + // The exported constructors take a map the caller may have decoded with + // UseNumber, and must read a number there as text. decoder := json.NewDecoder(strings.NewReader(`{"age": 12, "name": "Lukla"}`)) decoder.UseNumber() diff --git a/go/flightradarapi/testdata/airlines.html b/go/flightradarapi/testdata/airlines.html index 3c56b99..020dfd5 100644 --- a/go/flightradarapi/testdata/airlines.html +++ b/go/flightradarapi/testdata/airlines.html @@ -1,60 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- LATAM Airlines - xxLA / LAN340 aircraft
- Gol - xxG3 / GLO140 aircraft
- Delta Air Lines - xxDL / DAL900 aircraft
- Sky2 - xxSK10 aircraft
- SkyTeam - xxSKT15 aircraft
placeholder
- Should Skip -
- + + +Airlines - Flightradar24 + +
+
+

Largest fleets

+ All +
+ +
+
+

All airlines

+ +
+ + diff --git a/go/flightradarapi/testdata/airports.json b/go/flightradarapi/testdata/airports.json deleted file mode 100644 index 924f9db..0000000 --- a/go/flightradarapi/testdata/airports.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1786010717", - "rows": [ - { - "name": "Sao Paulo Guarulhos International Airport", - "iata": "GRU", - "icao": "SBGR", - "lat": -23.429991, - "lon": -46.4674, - "country": "Brazil", - "alt": 2436 - }, - { - "name": "Rio de Janeiro Galeao International Airport", - "iata": "GIG", - "icao": "SBGL", - "lat": -22.805696, - "lon": -43.25523, - "country": "Brazil", - "alt": 22 - }, - { - "name": "Sao Paulo Congonhas Airport", - "iata": "CGH", - "icao": "SBSP", - "lat": -23.627348, - "lon": -46.655987, - "country": "Brazil", - "alt": 2614 - }, - { - "name": "Los Angeles International Airport", - "iata": "LAX", - "icao": "KLAX", - "lat": 33.94252, - "lon": -118.406998, - "country": "United States", - "alt": 125 - }, - { - "name": "New York John F. Kennedy International Airport", - "iata": "JFK", - "icao": "KJFK", - "lat": 40.639751, - "lon": -73.7789, - "country": "United States", - "alt": 14 - }, - { - "name": "A Coruna Airport", - "iata": "LCG", - "icao": "LECO", - "lat": 43.302059, - "lon": -8.37725, - "country": "Spain", - "alt": 326 - }, - { - "name": "Ann Airport", - "iata": "VBA", - "icao": "VYAN", - "lat": 19.770643, - "lon": 94.026405, - "country": "Myanmar (Burma)", - "alt": "43" - }, - { - "name": "Broken Coords Airport", - "iata": "BAD", - "icao": "SBAD", - "lat": "n/a", - "lon": "", - "country": "Brazil", - "alt": 0 - } - ] -} diff --git a/go/flightradarapi/testdata/airports_country.html b/go/flightradarapi/testdata/airports_country.html new file mode 100644 index 0000000..833d4f4 --- /dev/null +++ b/go/flightradarapi/testdata/airports_country.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/go/flightradarapi/testdata/airports_country.json b/go/flightradarapi/testdata/airports_country.json new file mode 100644 index 0000000..09b4811 --- /dev/null +++ b/go/flightradarapi/testdata/airports_country.json @@ -0,0 +1,51 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "uruguay" + }, + "query": [] + }, + "country": { + "id": 238, + "name": "Uruguay", + "slug": "uruguay" + }, + "airports": [ + { + "id": 787, + "name": "Colonia Laguna de Los Patos International Airport", + "city": "Colonia", + "iata": "CYR", + "icao": "SUCA", + "total": 8 + }, + { + "id": 2349, + "name": "Montevideo Carrasco International Airport", + "city": "Montevideo", + "iata": "MVD", + "icao": "SUMU", + "total": 632 + } + ], + "airportImages": { + "SUMU": { + "thumbnail": "https://cdn.jetphotos.com/200/5/1.jpg", + "copyright": "Someone" + } + }, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/uruguay", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/flightradarapi/testdata/airports_country_states.json b/go/flightradarapi/testdata/airports_country_states.json new file mode 100644 index 0000000..ff8bca1 --- /dev/null +++ b/go/flightradarapi/testdata/airports_country_states.json @@ -0,0 +1,41 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "brazil" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "states": [ + { + "code": "RJ", + "name": "Rio de Janeiro", + "total": 1, + "url": "/data/airports/brazil/rj" + }, + { + "code": "SP", + "name": "Sao Paulo", + "total": 2, + "url": "/data/airports/brazil/sp" + } + ], + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/flightradarapi/testdata/airports_index.html b/go/flightradarapi/testdata/airports_index.html new file mode 100644 index 0000000..a55ed2f --- /dev/null +++ b/go/flightradarapi/testdata/airports_index.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/go/flightradarapi/testdata/airports_state_rj.json b/go/flightradarapi/testdata/airports_state_rj.json new file mode 100644 index 0000000..77d7a6b --- /dev/null +++ b/go/flightradarapi/testdata/airports_state_rj.json @@ -0,0 +1,44 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "rj" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "RJ", + "name": "Rio de Janeiro", + "slug": "rj" + }, + "airports": [ + { + "id": 1252, + "name": "Rio de Janeiro Galeao International Airport", + "city": "Rio de Janeiro", + "iata": "GIG", + "icao": "SBGL", + "total": 2500 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/rj", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/go/flightradarapi/testdata/airports_state_sp.json b/go/flightradarapi/testdata/airports_state_sp.json new file mode 100644 index 0000000..46c60c6 --- /dev/null +++ b/go/flightradarapi/testdata/airports_state_sp.json @@ -0,0 +1,52 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "sp" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "SP", + "name": "Sao Paulo", + "slug": "sp" + }, + "airports": [ + { + "id": 1255, + "name": "Sao Paulo Guarulhos International Airport", + "city": "Sao Paulo", + "iata": "GRU", + "icao": "SBGR", + "total": 6796 + }, + { + "id": 1254, + "name": "Sao Paulo Congonhas Airport", + "city": "Sao Paulo", + "iata": "CGH", + "icao": "SBSP", + "total": 3100 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/sp", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/FlightRadarAPI/api.js b/nodejs/FlightRadarAPI/api.js index 6d50ebf..00eb664 100644 --- a/nodejs/FlightRadarAPI/api.js +++ b/nodejs/FlightRadarAPI/api.js @@ -5,7 +5,7 @@ const Flight = require("./entities/flight"); const FlightTrackerConfig = require("./flightTrackerConfig"); const { AirportNotFoundError, LoginError } = require("./errors"); const { isNumeric, radians, rad2deg } = require("./util"); -const { parseAirlinesHtml, parseAirportsJson, countryToSlug } = require("./parsers"); +const { parseAirlinesHtml, parseAirportsIndex, parseAirportsPage, parseInertiaPage, countryToSlug } = require("./parsers"); /** @@ -200,8 +200,14 @@ class FlightRadar24API { /** * Return a list with all airports, optionally narrowed to some countries. * + * FR24 lists airports one page per country, and one per state for the + * largest countries, so this fetches one page per country asked for (a few + * hundred for every airport), `maxWorkers` at a time. The listing carries + * the name, codes and country only: use getAirport(code) for the position + * and altitude of an airport. + * * @param {Iterable|string} [countries] - Country names from the Countries enum, - * as any iterable or a single value; every country when omitted + * or their slug strings, as any iterable or a single value; every country when omitted * @return {Promise>} */ async getAirports(countries) { @@ -222,10 +228,94 @@ class FlightRadar24API { if (wanted && wanted.length === 0) return []; + // The index names every country, and declares the asset version the + // JSON requests below have to quote. const { content } = await this.__client.request( - Core.airportsJsonUrl, { headers: Core.jsonHeaders, timeout: this.timeout }, + Core.airportsDataUrl, { headers: Core.htmlHeaders, timeout: this.timeout }, + ); + const index = parseInertiaPage(content); + const version = typeof index?.["version"] === "string" ? index["version"] : null; + + const slugs = wanted === null ? + parseAirportsIndex(index ?? {}).map((country) => country.slug) : + wanted.map(countryToSlug); + const uniqueSlugs = [...new Set(slugs.filter((slug) => slug))]; + + const countryPages = await this.__getAirportPages(uniqueSlugs.map(Core.airportsByCountryUrl), version); + + // Second pass for the countries split by state, every state at once. + const stateOwners = []; + const stateUrls = []; + + countryPages.forEach(({ statePaths }, owner) => { + for (const path of statePaths) { + stateOwners.push(owner); + stateUrls.push(path.startsWith("http") ? path : Core.flightRadarBaseUrl + path); + } + }); + + const statePages = await this.__getAirportPages(stateUrls, version); + + // Grouped by country, in the order asked for, each country's states after it. + const airportsByCountry = countryPages.map(({ airports }) => [...airports]); + + statePages.forEach(({ airports }, i) => airportsByCountry[stateOwners[i]].push(...airports)); + + // A country FR24 does not know answers with the index page, which lists + // neither airports nor states. + const missing = uniqueSlugs.filter( + (slug, i) => countryPages[i].airports.length === 0 && countryPages[i].statePaths.length === 0, + ); + + if (missing.length > 0) { + console.warn(`getAirports: no airports found for ${missing.join(", ")} — check the Countries enum.`); + } + + return airportsByCountry.flat(); + } + + /** + * Fetch and parse the airport listing pages at `urls`, in order, maxWorkers at a time. + * + * @param {Array} urls + * @param {string|null} version - Asset version declared by the index page + * @return {Promise, statePaths: Array}>>} + */ + async __getAirportPages(urls, version) { + const pages = new Array(urls.length); + + await mapConcurrent(urls.map((url, i) => [url, i]), this.maxWorkers, async ([url, i]) => { + pages[i] = parseAirportsPage(await this.__getInertiaPage(url, version)); + }); + + return pages; + } + + /** + * Fetch a FR24 data page as its Inertia page object, falling back to the HTML. + * + * With the asset version FR24 answers the bare page JSON, a fraction of the + * HTML. A 409 means the assets were redeployed since the version was read: + * the HTML carries the same page object and needs no version. + * + * @param {string} url + * @param {string|null} version + * @return {Promise} + */ + async __getInertiaPage(url, version) { + if (version !== null) { + const headers = { ...Core.inertiaHeaders, "x-inertia-version": version }; + const { content, statusCode } = await this.__client.requestStandalone( + url, { headers, timeout: this.timeout, allowedErrorCodes: [409] }, + ); + + if (statusCode !== 409) return content; + } + + const { content } = await this.__client.requestStandalone( + url, { headers: Core.htmlHeaders, timeout: this.timeout }, ); - return parseAirportsJson(content, wanted); + return content; } /** diff --git a/nodejs/FlightRadarAPI/core.js b/nodejs/FlightRadarAPI/core.js index 638362c..ae8f6f6 100644 --- a/nodejs/FlightRadarAPI/core.js +++ b/nodejs/FlightRadarAPI/core.js @@ -46,9 +46,12 @@ const Core = { apiAirportDataUrl: `${API_FR24_BASE}/airport.json`, airportDataUrl: (code) => `${FR24_BASE}/airports/traffic-stats/?airport=${code}`, + // The listing is split by country, and by state for the largest countries. + // These are Inertia pages: the data rides JSON-encoded in the data-page + // attribute of the HTML, or comes as the whole body when the request carries + // `inertiaHeaders` and the asset version the index page declares. airportsDataUrl: `${FR24_BASE}/data/airports`, - // Every airport in one payload; the per-country pages above are client-rendered. - airportsJsonUrl: `${FR24_BASE}/_json/airports.php`, + airportsByCountryUrl: (slug) => `${FR24_BASE}/data/airports/${slug}`, airlinesDataUrl: `${FR24_BASE}/data/airlines`, @@ -72,6 +75,11 @@ const Core = { headers: baseHeaders, jsonHeaders: { accept: "application/json", ...baseHeaders }, imageHeaders: { accept: "image/gif, image/jpg, image/jpeg, image/png", ...baseHeaders }, + // Asks an Inertia page for its page object instead of the HTML around it. + // `x-inertia-version` must match the deployed assets or FR24 answers 409. + inertiaHeaders: { + "accept": "application/json", ...baseHeaders, "x-inertia": "true", "x-requested-with": "XMLHttpRequest", + }, htmlHeaders: { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "accept-encoding": "gzip, deflate, br", diff --git a/nodejs/FlightRadarAPI/index.d.ts b/nodejs/FlightRadarAPI/index.d.ts index 533d408..1af0f81 100644 --- a/nodejs/FlightRadarAPI/index.d.ts +++ b/nodejs/FlightRadarAPI/index.d.ts @@ -116,8 +116,11 @@ export class FlightRadar24API { /** * Return a list with all airports, optionally narrowed to some countries. * - * @param {Iterable|string} [countries] - Country names, as any iterable - * or a single value; every country when omitted + * The listing carries the name, codes and country only: latitude, longitude + * and altitude are null. Use getAirport(code) for the position of an airport. + * + * @param {Iterable|string} [countries] - Country names from the Countries enum, + * or their slug strings, as any iterable or a single value; every country when omitted */ getAirports(countries?: Iterable | string): Promise; diff --git a/nodejs/FlightRadarAPI/parsers.js b/nodejs/FlightRadarAPI/parsers.js index 79a16dd..2e0c396 100644 --- a/nodejs/FlightRadarAPI/parsers.js +++ b/nodejs/FlightRadarAPI/parsers.js @@ -1,73 +1,114 @@ const { parse } = require("node-html-parser"); const Airport = require("./entities/airport"); +// An airline code badge: a two-letter IATA code or a three-letter ICAO code. +// ASCII only, so a stray Unicode letter does not pass for a code. +const AIRLINE_CODE_PATTERN = /^[A-Z0-9]{2,3}$/; + +// A fleet size as the page prints it: "340" or "3,878". +const AIRCRAFT_COUNT_PATTERN = /^\d{1,3}(?:[,.]\d{3})*$|^\d+$/; + +const AIRLINES_PATH_PREFIX = "/data/airlines/"; + +/** + * Whether an element holds no other element (text and comments only). + * + * @param {HTMLElement} element + * @return {boolean} + */ +function isLeaf(element) { + return element.childNodes.every((node) => node.nodeType !== 1); +} + +/** + * Text of the `tag` elements under `root` that hold no other element. + * + * The name element is left out by its `title` attribute: an airline called + * "TAP" would otherwise read as an ICAO code. + * + * @param {HTMLElement} root + * @param {string} tag + * @return {Array} + */ +function leafTexts(root, tag) { + return root.querySelectorAll(tag) + .filter((element) => isLeaf(element) && !element.hasAttribute("title")) + .map((element) => element.text.trim()); +} + +/** + * The airline name of a listing entry. + * + * The name sits in the element titled with it, with the logo's aria-label as a + * fallback; the anchor text also carries the codes and the fleet size. + * + * @param {HTMLElement} link + * @return {string} + */ +function airlineName(link) { + for (const attribute of ["title", "aria-label"]) { + const element = link.querySelector(`[${attribute}]`); + + if (element) return String(element.getAttribute(attribute) ?? "").trim(); + } + + return link.text.trim(); +} + /** - * Parse the airlines listing HTML page into a list of airline objects. + * Parse the airlines listing page into a list of airline objects. + * + * Every airline is a link to its data page, with the IATA and ICAO codes as + * badges next to the name. The "Largest fleets" section repeats a few airlines + * with their aircraft count; an airline the page lists twice becomes one entry + * carrying everything the page said about it. * * @param {string|Buffer} html * @return {Array} */ function parseAirlinesHtml(html) { - const root = parse(typeof html === "string" ? html : html.toString()); - const tbody = root.querySelector("tbody"); + const root = parse(toText(html) ?? ""); + const links = root.querySelectorAll("a[href]") + .filter((link) => String(link.getAttribute("href")).startsWith(AIRLINES_PATH_PREFIX)); - if (!tbody) { - console.warn("parseAirlinesHtml: no in response — FR24 page layout may have changed."); + if (links.length === 0) { + console.warn("parseAirlinesHtml: no airline links in response — FR24 page layout may have changed."); return []; } - const airlines = []; - - for (const tr of tbody.querySelectorAll("tr")) { - const tdNotranslate = tr.querySelector("td.notranslate"); - - if (!tdNotranslate) continue; + // Keyed by href, so an airline listed in two sections becomes one entry. + const airlines = new Map(); - const aElement = tdNotranslate.querySelector("a[href^='/data/airlines']"); + for (const link of links) { + const name = airlineName(link); - if (!aElement) continue; + if (name.length < 2) continue; - const airlineName = aElement.text.trim(); + const codes = leafTexts(link, "span").filter((text) => AIRLINE_CODE_PATTERN.test(text)); + const iata = codes.find((code) => code.length === 2) ?? null; + const icao = codes.find((code) => code.length === 3) ?? null; - if (airlineName.length < 2) continue; + // Section links ("All") point under /data/airlines/ too; a code tells + // an airline apart from them. + if (iata === null && icao === null) continue; - const tdElements = tr.querySelectorAll("td"); - let iata = null; - let icao = null; + const countText = leafTexts(link, "strong").find((text) => AIRCRAFT_COUNT_PATTERN.test(text)); + const nAircrafts = countText === undefined ? null : parseInt(countText.replace(/\D/g, ""), 10); - if (tdElements.length >= 4) { - const codesText = tdElements[3].text.trim(); + const href = link.getAttribute("href"); - if (codesText.includes(" / ")) { - const parts = codesText.split(" / "); - - if (parts.length === 2) { - iata = parts[0].trim(); - icao = parts[1].trim(); - } - } - else if (codesText.length === 2) { - iata = codesText; - } - else if (codesText.length === 3) { - icao = codesText; - } + if (!airlines.has(href)) { + airlines.set(href, { "Name": name, "ICAO": null, "IATA": null, "n_aircrafts": null }); } - let nAircrafts = null; - - if (tdElements.length >= 5) { - const aircraftsText = tdElements[4].text.trim(); + const airline = airlines.get(href); - if (aircraftsText) { - nAircrafts = parseInt(aircraftsText.split(" ")[0].trim(), 10); - } + for (const [key, value] of [["ICAO", icao], ["IATA", iata], ["n_aircrafts", nAircrafts]]) { + if (airline[key] === null) airline[key] = value; } - - airlines.push({ "Name": airlineName, "ICAO": icao, "IATA": iata, "n_aircrafts": nAircrafts }); } - return airlines; + return [...airlines.values()]; } /** @@ -82,35 +123,10 @@ function countryToSlug(country) { // Punctuation becomes a hyphen rather than being deleted: FR24's own assets // are named that way, e.g. flags-small/cote-d-ivoire.svg. return String(country ?? "") - .normalize("NFKD").replace(/[\u0300-\u036f]/g, "") + .normalize("NFKD").replace(/[̀-ͯ]/g, "") .toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, ""); } -// ASCII decimals only, so both ports accept and reject the same strings. -const NUMERIC_PATTERN = /^[+-]?([0-9]+\.?[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?$/; - -// trim() and str.strip() disagree on U+FEFF and U+001C-U+001F: pick one set. -const SURROUNDING_SPACE = /^[ \t\n\r\f\v]+|[ \t\n\r\f\v]+$/g; - -/** - * Coerce a numeric feed field into a number, or null when it is unusable. - * - * An unusable coordinate must not become 0: that would place the airport in the - * Gulf of Guinea instead of marking its position as unknown. - * - * @param {*} value - * @return {number|null} - */ -function toNumber(value) { - if (typeof value === "number") return Number.isFinite(value) ? value : null; - // Not stringified: String([43]) would pass as 43. - if (typeof value !== "string") return null; - - const text = value.replace(SURROUNDING_SPACE, ""); - - return NUMERIC_PATTERN.test(text) ? toNumber(Number(text)) : null; -} - /** * Keep a text field as a string, or "" when the feed sends anything else. * @@ -144,90 +160,127 @@ function toText(payload) { } /** - * Parse the airports JSON feed into a list of Airport instances. + * Whether a value is a plain object: arrays are objects in JS, and Python's + * isinstance(value, dict) drops them. + * + * @param {*} value + * @return {boolean} + */ +function isObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** + * Return the Inertia page object (component, props, version) of a FR24 data page. + * + * Served as HTML, the page keeps it JSON-encoded in the data-page attribute of + * its root element; asked with Core.inertiaHeaders, FR24 sends it as the body, + * which `request()` has already decoded into an object. * - * @param {object|string|Buffer} payload - Body of Core.airportsJsonUrl - * @param {Array} [countries] - Country slugs from the Countries enum; every airport is kept when omitted - * @return {Array} + * @param {object|string|Buffer|ArrayBuffer} payload + * @return {object|null} */ -function parseAirportsJson(payload, countries = null) { - let data = payload; +function parseInertiaPage(payload) { + let page = payload; const text = toText(payload); if (text !== null) { + // node-html-parser decodes the attribute's entities (", '...). + const root = parse(text).querySelector("[data-page]"); + + if (!root) { + console.warn("parseInertiaPage: no data-page attribute in response — FR24 page layout may have changed."); + return null; + } + try { - data = JSON.parse(text); + page = JSON.parse(root.getAttribute("data-page")); } catch { - console.warn("parseAirportsJson: response is not valid JSON — FR24 feed may have changed."); - return []; + console.warn("parseInertiaPage: data-page is not valid JSON — FR24 page layout may have changed."); + return null; } } - const rows = data && Array.isArray(data["rows"]) ? data["rows"] : null; + if (!isObject(page) || !isObject(page["props"])) { + console.warn("parseInertiaPage: no page props in response — FR24 page layout may have changed."); + return null; + } + + return page; +} - if (!rows) { - console.warn("parseAirportsJson: no \"rows\" array in response — FR24 feed may have changed."); +/** + * Parse the airports index (Core.airportsDataUrl) into its countries. + * + * Each entry has the country "name" as FR24 prints it ("UNITED STATES") and the + * "slug" of its listing page, which is also its Countries enum value. + * + * @param {object|string|Buffer|ArrayBuffer} payload + * @return {Array<{name: string, slug: string}>} + */ +function parseAirportsIndex(payload) { + const page = parseInertiaPage(payload); + const rows = page !== null ? page["props"]["airportsByCountry"] : null; + + if (!Array.isArray(rows)) { + console.warn("parseAirportsIndex: no \"airportsByCountry\" list in response — FR24 page layout may have changed."); return []; } - const wanted = countries ? new Set(countries.map(countryToSlug)) : null; - const matched = new Set(); - const unpositioned = []; - const airports = []; + const countries = []; for (const row of rows) { - // Arrays are objects in JS; Python's isinstance(row, dict) drops them. - if (!row || typeof row !== "object" || Array.isArray(row)) continue; + if (!isObject(row)) continue; - // Slugified only when filtering: it is otherwise unused, and this loop - // runs over every airport in the feed. - if (wanted) { - const slug = countryToSlug(row["country"]); + const name = toStringField(row["name"]); + const slug = countryToSlug(name); - if (!wanted.has(slug)) continue; - matched.add(slug); - } + if (slug) countries.push({ name, slug }); + } - let latitude = toNumber(row["lat"]); - let longitude = toNumber(row["lon"]); + return countries; +} - // One bad coordinate drops both: half a position reads as located. - if (latitude === null || longitude === null) { - latitude = null; - longitude = null; - unpositioned.push(toStringField(row["name"])); - } +/** + * Parse a country or state airports page into its airports and state pages. + * + * A country with many airports lists its states instead, each with the path of + * its own page; those paths come back for the caller to fetch. The listing has + * no coordinates or altitude: `getAirport(code)` has them for one airport. + * + * @param {object|string|Buffer|ArrayBuffer} payload + * @return {{airports: Array, statePaths: Array}} + */ +function parseAirportsPage(payload) { + const page = parseInertiaPage(payload); - airports.push(new Airport({ - "name": toStringField(row["name"]), - "icao": toStringField(row["icao"]), - "iata": toStringField(row["iata"]), - "lat": latitude, - "lon": longitude, - "alt": toNumber(row["alt"]), - "country": toStringField(row["country"]), - })); + if (page === null) { + return { airports: [], statePaths: [] }; } - // One line, not one per row: the feed carries every airport. - if (unpositioned.length > 0) { - const sample = unpositioned.slice(0, 3).map((name) => `"${name}"`).join(", "); - console.warn( - `parseAirportsJson: ${unpositioned.length} airport(s) had unusable coordinates ` + - `and carry no position (e.g. ${sample}).`, - ); - } + const props = page["props"]; + const country = props["country"]; + const countryName = isObject(country) ? toStringField(country["name"]) : ""; - if (wanted) { - const missing = [...wanted].filter((slug) => !matched.has(slug)); + const rows = Array.isArray(props["airports"]) ? props["airports"] : []; + const states = Array.isArray(props["states"]) ? props["states"] : []; - if (missing.length > 0) { - console.warn(`parseAirportsJson: no airports found for ${missing.join(", ")} — check the Countries enum.`); - } - } + const airports = rows.filter(isObject).map((row) => new Airport({ + "name": toStringField(row["name"]), + "icao": toStringField(row["icao"]), + "iata": toStringField(row["iata"]), + "lat": null, + "lon": null, + "alt": null, + "country": countryName, + })); + + const statePaths = states + .filter((state) => isObject(state) && typeof state["url"] === "string" && state["url"] !== "") + .map((state) => state["url"]); - return airports; + return { airports, statePaths }; } -module.exports = { parseAirlinesHtml, parseAirportsJson, countryToSlug }; +module.exports = { parseAirlinesHtml, parseInertiaPage, parseAirportsIndex, parseAirportsPage, countryToSlug }; diff --git a/nodejs/package.json b/nodejs/package.json index e6efbcf..2a5f367 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -6,7 +6,7 @@ "types": "./FlightRadarAPI/index.d.ts", "scripts": { "test": "mocha tests --timeout 10000", - "test:offline": "mocha tests/testParsersOffline.js tests/testRequestPolicy.js tests/testRequestTransport.js tests/testFeedRetry.js --timeout 10000", + "test:offline": "mocha tests/testParsersOffline.js tests/testAirportsOffline.js tests/testRequestPolicy.js tests/testRequestTransport.js tests/testFeedRetry.js --timeout 10000", "test:integration": "mocha tests/testApi.js tests/testSnapshots.js --timeout 10000", "test:types": "tsd", "lint": "eslint ." diff --git a/nodejs/tests/fixtures/airlines.html b/nodejs/tests/fixtures/airlines.html index 3c56b99..020dfd5 100644 --- a/nodejs/tests/fixtures/airlines.html +++ b/nodejs/tests/fixtures/airlines.html @@ -1,60 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- LATAM Airlines - xxLA / LAN340 aircraft
- Gol - xxG3 / GLO140 aircraft
- Delta Air Lines - xxDL / DAL900 aircraft
- Sky2 - xxSK10 aircraft
- SkyTeam - xxSKT15 aircraft
placeholder
- Should Skip -
- + + +Airlines - Flightradar24 + +
+
+

Largest fleets

+ All +
+ +
+
+

All airlines

+ +
+ + diff --git a/nodejs/tests/fixtures/airports.json b/nodejs/tests/fixtures/airports.json deleted file mode 100644 index 924f9db..0000000 --- a/nodejs/tests/fixtures/airports.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1786010717", - "rows": [ - { - "name": "Sao Paulo Guarulhos International Airport", - "iata": "GRU", - "icao": "SBGR", - "lat": -23.429991, - "lon": -46.4674, - "country": "Brazil", - "alt": 2436 - }, - { - "name": "Rio de Janeiro Galeao International Airport", - "iata": "GIG", - "icao": "SBGL", - "lat": -22.805696, - "lon": -43.25523, - "country": "Brazil", - "alt": 22 - }, - { - "name": "Sao Paulo Congonhas Airport", - "iata": "CGH", - "icao": "SBSP", - "lat": -23.627348, - "lon": -46.655987, - "country": "Brazil", - "alt": 2614 - }, - { - "name": "Los Angeles International Airport", - "iata": "LAX", - "icao": "KLAX", - "lat": 33.94252, - "lon": -118.406998, - "country": "United States", - "alt": 125 - }, - { - "name": "New York John F. Kennedy International Airport", - "iata": "JFK", - "icao": "KJFK", - "lat": 40.639751, - "lon": -73.7789, - "country": "United States", - "alt": 14 - }, - { - "name": "A Coruna Airport", - "iata": "LCG", - "icao": "LECO", - "lat": 43.302059, - "lon": -8.37725, - "country": "Spain", - "alt": 326 - }, - { - "name": "Ann Airport", - "iata": "VBA", - "icao": "VYAN", - "lat": 19.770643, - "lon": 94.026405, - "country": "Myanmar (Burma)", - "alt": "43" - }, - { - "name": "Broken Coords Airport", - "iata": "BAD", - "icao": "SBAD", - "lat": "n/a", - "lon": "", - "country": "Brazil", - "alt": 0 - } - ] -} diff --git a/nodejs/tests/fixtures/airports_country.html b/nodejs/tests/fixtures/airports_country.html new file mode 100644 index 0000000..833d4f4 --- /dev/null +++ b/nodejs/tests/fixtures/airports_country.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/nodejs/tests/fixtures/airports_country.json b/nodejs/tests/fixtures/airports_country.json new file mode 100644 index 0000000..09b4811 --- /dev/null +++ b/nodejs/tests/fixtures/airports_country.json @@ -0,0 +1,51 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "uruguay" + }, + "query": [] + }, + "country": { + "id": 238, + "name": "Uruguay", + "slug": "uruguay" + }, + "airports": [ + { + "id": 787, + "name": "Colonia Laguna de Los Patos International Airport", + "city": "Colonia", + "iata": "CYR", + "icao": "SUCA", + "total": 8 + }, + { + "id": 2349, + "name": "Montevideo Carrasco International Airport", + "city": "Montevideo", + "iata": "MVD", + "icao": "SUMU", + "total": 632 + } + ], + "airportImages": { + "SUMU": { + "thumbnail": "https://cdn.jetphotos.com/200/5/1.jpg", + "copyright": "Someone" + } + }, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/uruguay", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/fixtures/airports_country_states.json b/nodejs/tests/fixtures/airports_country_states.json new file mode 100644 index 0000000..ff8bca1 --- /dev/null +++ b/nodejs/tests/fixtures/airports_country_states.json @@ -0,0 +1,41 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "brazil" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "states": [ + { + "code": "RJ", + "name": "Rio de Janeiro", + "total": 1, + "url": "/data/airports/brazil/rj" + }, + { + "code": "SP", + "name": "Sao Paulo", + "total": 2, + "url": "/data/airports/brazil/sp" + } + ], + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/fixtures/airports_index.html b/nodejs/tests/fixtures/airports_index.html new file mode 100644 index 0000000..a55ed2f --- /dev/null +++ b/nodejs/tests/fixtures/airports_index.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/nodejs/tests/fixtures/airports_state_rj.json b/nodejs/tests/fixtures/airports_state_rj.json new file mode 100644 index 0000000..77d7a6b --- /dev/null +++ b/nodejs/tests/fixtures/airports_state_rj.json @@ -0,0 +1,44 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "rj" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "RJ", + "name": "Rio de Janeiro", + "slug": "rj" + }, + "airports": [ + { + "id": 1252, + "name": "Rio de Janeiro Galeao International Airport", + "city": "Rio de Janeiro", + "iata": "GIG", + "icao": "SBGL", + "total": 2500 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/rj", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/fixtures/airports_state_sp.json b/nodejs/tests/fixtures/airports_state_sp.json new file mode 100644 index 0000000..46c60c6 --- /dev/null +++ b/nodejs/tests/fixtures/airports_state_sp.json @@ -0,0 +1,52 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "sp" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "SP", + "name": "Sao Paulo", + "slug": "sp" + }, + "airports": [ + { + "id": 1255, + "name": "Sao Paulo Guarulhos International Airport", + "city": "Sao Paulo", + "iata": "GRU", + "icao": "SBGR", + "total": 6796 + }, + { + "id": 1254, + "name": "Sao Paulo Congonhas Airport", + "city": "Sao Paulo", + "iata": "CGH", + "icao": "SBSP", + "total": 3100 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/sp", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/nodejs/tests/testAirportsOffline.js b/nodejs/tests/testAirportsOffline.js new file mode 100644 index 0000000..27f2f0d --- /dev/null +++ b/nodejs/tests/testAirportsOffline.js @@ -0,0 +1,202 @@ +/** + * Offline tests for the page crawl behind getAirports(). + * + * FR24 lists airports one Inertia page per country, and per state for the + * largest countries. These tests pin down which pages the crawl asks for and + * how it reads them, against the bundled fixtures, so the PR gate covers the + * orchestration without reaching FR24. + */ +const fs = require("fs"); +const path = require("path"); +const expect = require("chai").expect; + +const { FlightRadar24API, Countries } = require("../FlightRadarAPI/index"); +const Core = require("../FlightRadarAPI/core"); +const { parseInertiaPage } = require("../FlightRadarAPI/parsers"); + +const FIXTURES = path.join(__dirname, "fixtures"); +const load = (name) => fs.readFileSync(path.join(FIXTURES, name), "utf-8"); + +const INDEX_VERSION = "919b94d2a33f365a34f99828f6eeec6a"; + +// Every page the fixtures model, by the path FR24 serves it at. +const PAGES = { + "/data/airports": "airports_index.html", + "/data/airports/uruguay": "airports_country.json", + "/data/airports/brazil": "airports_country_states.json", + "/data/airports/brazil/sp": "airports_state_sp.json", + "/data/airports/brazil/rj": "airports_state_rj.json", + "/data/airports/myanmar-burma": "airports_country.html", +}; + +/** + * The Inertia page object of a fixture, as the request layer would decode a JSON body. + * + * @param {string} fixture + * @return {object} + */ +function pageOf(fixture) { + if (fixture.endsWith(".json")) return JSON.parse(load(fixture)); + + const page = parseInertiaPage(load(fixture)); + expect(page).to.not.equal(null); + return page; +} + +/** + * Swap in a client double that serves the fixture pages the way FR24 does. + * + * A request carrying the inertia headers gets the page JSON when the version + * matches and a 409 otherwise; a plain request gets the HTML. The index page + * only exists as HTML, like the real one as the crawl fetches it. + * + * @param {string|null} [version] - Asset version the fake deployment runs + * @return {{api: FlightRadar24API, calls: Array, paths: Function}} + */ +function apiWithFixturePages(version = INDEX_VERSION) { + const api = new FlightRadar24API(); + const calls = []; + + const serve = async (url, options = {}) => { + calls.push({ url, options }); + const pagePath = url.replace(Core.flightRadarBaseUrl, ""); + let fixture = PAGES[pagePath] ?? "airports_index.html"; + const headers = options.headers ?? {}; + + if (headers["x-inertia"] === "true") { + if (headers["x-inertia-version"] !== version) { + expect(options.allowedErrorCodes ?? [], "a 409 must not raise").to.include(409); + return { content: "", statusCode: 409, cookies: {} }; + } + return { content: pageOf(fixture), statusCode: 200, cookies: {} }; + } + + if (fixture.endsWith(".json")) { + fixture = fixture.replace(".json", ".html"); + + if (!fs.existsSync(path.join(FIXTURES, fixture))) fixture = "airports_country.html"; + } + return { content: load(fixture), statusCode: 200, cookies: {} }; + }; + + api.__client = { request: serve, requestStandalone: serve }; + + const paths = () => calls.map((call) => call.url.replace(Core.flightRadarBaseUrl, "")); + + return { api, calls, paths }; +} + +/** + * Run fn while capturing console.warn, returning what it printed. + * + * @param {Function} fn + * @return {Promise>} + */ +async function capturingWarnings(fn) { + const warnings = []; + const original = console.warn; + console.warn = (...args) => warnings.push(args.join(" ")); + + try { + await fn(); + } + finally { + console.warn = original; + } + return warnings; +} + + +describe("getAirports() page crawl (offline)", function() { + it("crawls the country, then its states", async function() { + const { api, paths } = apiWithFixturePages(); + + const airports = await api.getAirports([Countries.BRAZIL]); + + expect(airports.map((a) => a.iata)).to.deep.equal(["GIG", "GRU", "CGH"]); + expect(new Set(airports.map((a) => a.country))).to.deep.equal(new Set(["Brazil"])); + expect(paths()).to.deep.equal([ + "/data/airports", "/data/airports/brazil", "/data/airports/brazil/rj", "/data/airports/brazil/sp", + ]); + }); + + it("asks for the page JSON with the index version", async function() { + const { api, calls } = apiWithFixturePages(); + + await api.getAirports(Countries.URUGUAY); + + const [index, country] = calls; + expect(index.options.headers).to.not.have.property("x-inertia"); + expect(country.options.headers["x-inertia"]).to.equal("true"); + expect(country.options.headers["x-inertia-version"]).to.equal(INDEX_VERSION); + }); + + it("falls back to the HTML when the assets were redeployed", async function() { + const { api, calls, paths } = apiWithFixturePages("something-newer"); + + const airports = await api.getAirports(["uruguay"]); + + // The index answered with a stale version, the JSON request got a 409 + // and the HTML page was read instead. + expect(airports).to.have.lengthOf(1); + expect(paths()).to.deep.equal(["/data/airports", "/data/airports/uruguay", "/data/airports/uruguay"]); + expect(calls[2].options.headers).to.deep.equal(Core.htmlHeaders); + }); + + it("walks the whole index when no country is given", async function() { + const { api, paths } = apiWithFixturePages(); + + const airports = await api.getAirports(); + + expect(airports.map((a) => a.iata)).to.deep.equal(["GIG", "GRU", "CGH", "RGN", "CYR", "MVD"]); + expect(paths()).to.deep.equal([ + "/data/airports", + "/data/airports/brazil", "/data/airports/myanmar-burma", "/data/airports/uruguay", + "/data/airports/brazil/rj", "/data/airports/brazil/sp", + ]); + }); + + it("accepts mixed spellings and drops repeats", async function() { + const { api, paths } = apiWithFixturePages(); + + const airports = await api.getAirports(["Uruguay", Countries.URUGUAY, "uruguay"]); + + expect(airports.map((a) => a.iata)).to.deep.equal(["CYR", "MVD"]); + expect(paths()).to.deep.equal(["/data/airports", "/data/airports/uruguay"]); + }); + + it("makes no request for an empty filter", async function() { + const { api, calls } = apiWithFixturePages(); + + expect(await api.getAirports([])).to.deep.equal([]); + expect(await api.getAirports(new Set())).to.deep.equal([]); + expect(calls).to.deep.equal([]); + }); + + it("rejects a non-iterable filter", async function() { + const { api, calls } = apiWithFixturePages(); + + let error = null; + try { + await api.getAirports(42); + } + catch (caught) { + error = caught; + } + + expect(error).to.be.an.instanceOf(TypeError); + expect(calls).to.deep.equal([]); + }); + + it("returns nothing and warns for an unknown country", async function() { + const { api } = apiWithFixturePages(); + let airports; + + const warnings = await capturingWarnings(async () => { + airports = await api.getAirports(["atlantis"]); + }); + + expect(airports).to.deep.equal([]); + expect(warnings.join("\n")).to.include("no airports found for atlantis"); + }); +}); diff --git a/nodejs/tests/testParsersOffline.js b/nodejs/tests/testParsersOffline.js index 0326870..4e3eb5d 100644 --- a/nodejs/tests/testParsersOffline.js +++ b/nodejs/tests/testParsersOffline.js @@ -11,29 +11,22 @@ const fs = require("fs"); const path = require("path"); const expect = require("chai").expect; -const { parseAirlinesHtml, parseAirportsJson, countryToSlug } = require("../FlightRadarAPI/parsers"); +const { + parseAirlinesHtml, parseInertiaPage, parseAirportsIndex, parseAirportsPage, countryToSlug, +} = require("../FlightRadarAPI/parsers"); const { Countries } = require("../FlightRadarAPI/core"); const FIXTURES = path.join(__dirname, "fixtures"); const load = (name) => fs.readFileSync(path.join(FIXTURES, name), "utf-8"); - -// Keep in step with the same list in python/tests/test_parsers_offline.py. -const NUMERIC_CASES = [ - ["whitespace", " ", null], - ["an array holding a number", [43], null], - ["a coordinate with a hemisphere suffix", "43.30 N", null], - ["an exponent overflowing a double", "1e999", null], - ["400 plain digits", "1".repeat(400), null], - ["a whole number as a string", "2436", 2436], - ["a negative decimal", "-23.4", -23.4], - ["a genuine zero", 0, 0], -]; +const loadJson = (name) => JSON.parse(load(name)); describe("parseAirlinesHtml (offline)", function() { let airlines; + let byName; before(function() { airlines = parseAirlinesHtml(load("airlines.html")); + byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); }); it("extracts known airline rows", function() { @@ -44,7 +37,6 @@ describe("parseAirlinesHtml (offline)", function() { }); it("splits IATA and ICAO when both are present", function() { - const byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); expect(byName["LATAM Airlines"].IATA).to.equal("LA"); expect(byName["LATAM Airlines"].ICAO).to.equal("LAN"); expect(byName["Gol"].IATA).to.equal("G3"); @@ -52,182 +44,167 @@ describe("parseAirlinesHtml (offline)", function() { }); it("handles IATA-only or ICAO-only rows", function() { - const byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); expect(byName["Sky2"].IATA).to.equal("SK"); expect(byName["Sky2"].ICAO).to.equal(null); expect(byName["SkyTeam"].ICAO).to.equal("SKT"); expect(byName["SkyTeam"].IATA).to.equal(null); }); - it("parses aircraft count from the last column", function() { - const byName = Object.fromEntries(airlines.map((a) => [a.Name, a])); + it("parses the aircraft count", function() { expect(byName["LATAM Airlines"].n_aircrafts).to.equal(340); expect(byName["Gol"].n_aircrafts).to.equal(140); + // Thousands separator, as the page prints large fleets. + expect(byName["Delta Air Lines"].n_aircrafts).to.equal(1900); + // Only the "Largest fleets" section carries a count. + expect(byName["Sky2"].n_aircrafts).to.equal(null); + }); + + it("merges an airline listed in two sections", function() { + // Delta and Gol appear under "Largest fleets" and again under "All airlines". + const names = airlines.map((a) => a.Name); + expect(names.filter((name) => name === "Delta Air Lines")).to.have.lengthOf(1); + expect(names.filter((name) => name === "Gol")).to.have.lengthOf(1); }); - it("skips rows without notranslate td or with wrong href", function() { - expect(airlines.length).to.equal(5); + it("keeps a code-shaped name", function() { + expect(byName["TAP"]).to.deep.equal({ "Name": "TAP", "ICAO": "TAP", "IATA": "TP", "n_aircrafts": null }); + }); + + it("skips invalid rows", function() { + // 6 airlines; the section link, the entry with no link, the wrong href + // and the one-letter name must be skipped. + expect(airlines).to.have.lengthOf(6); + expect(airlines.map((a) => a.Name)).to.not.include("All"); }); it("returns an empty list for empty or unrecognised HTML", function() { expect(parseAirlinesHtml("")).to.deep.equal([]); - expect(parseAirlinesHtml("

no tbody here

")).to.deep.equal([]); + expect(parseAirlinesHtml("

no airlines here

")).to.deep.equal([]); }); }); -describe("parseAirportsJson (offline)", function() { - let airports; - before(function() { - airports = parseAirportsJson(load("airports.json")); +describe("parseInertiaPage (offline)", function() { + it("reads the data-page attribute", function() { + const page = parseInertiaPage(load("airports_index.html")); + expect(page.component).to.equal("Data/Airports"); + expect(page.version).to.equal("919b94d2a33f365a34f99828f6eeec6a"); + expect(page.props).to.have.property("airportsByCountry"); }); - it("extracts iata, icao, name, country and position", function() { - const byIata = Object.fromEntries(airports.map((a) => [a.iata, a])); - expect(byIata["GRU"]).to.exist; - expect(byIata["GIG"]).to.exist; - expect(byIata["GRU"].icao).to.equal("SBGR"); - expect(byIata["GRU"].country).to.equal("Brazil"); - expect(byIata["GRU"].latitude).to.be.closeTo(-23.429991, 1e-6); - expect(byIata["GRU"].longitude).to.be.closeTo(-46.4674, 1e-6); - expect(byIata["GRU"].altitude).to.equal(2436); + it("accepts a decoded JSON body", function() { + const body = loadJson("airports_country.json"); + expect(parseInertiaPage(body)).to.equal(body); }); - it("keeps every country when no filter is given", function() { - const countries = new Set(airports.map((a) => a.country)); - expect(countries).to.include("Brazil"); - expect(countries).to.include("United States"); - expect(countries).to.include("Spain"); - }); + it("reads a body that arrived as raw bytes", function() { + // request() returns an ArrayBuffer on an unexpected content-type. + const html = load("airports_index.html"); - it("filters by the slugs from the Countries enum", function() { - const filtered = parseAirportsJson(load("airports.json"), ["united-states"]); - expect(filtered.length).to.be.above(0); - for (const airport of filtered) { - expect(airport.country).to.equal("United States"); + for (const payload of [Buffer.from(html), new TextEncoder().encode(html), new TextEncoder().encode(html).buffer]) { + expect(parseInertiaPage(payload).version, payload.constructor.name).to.equal("919b94d2a33f365a34f99828f6eeec6a"); } }); - it("accepts several countries at once", function() { - const filtered = parseAirportsJson(load("airports.json"), ["brazil", "spain"]); - const countries = new Set(filtered.map((a) => a.country)); - expect([...countries].sort()).to.deep.equal(["Brazil", "Spain"]); - }); - - it("invalid coordinates become null (regression test for 0,0 fallback)", function() { - const bad = airports.find((a) => a.iata === "BAD"); - expect(bad.latitude).to.equal(null); - expect(bad.longitude).to.equal(null); + it("rejects unrecognised payloads", function() { + expect(parseInertiaPage("")).to.equal(null); + expect(parseInertiaPage("
")).to.equal(null); + expect(parseInertiaPage("
")).to.equal(null); + expect(parseInertiaPage({ component: "Data/Airports" })).to.equal(null); }); +}); - it("keeps FR24's own country spelling and still slugifies it for URLs", function() { - const ann = airports.find((airport) => airport.iata === "VBA"); - expect(ann.country).to.equal("Myanmar (Burma)"); - expect(countryToSlug(ann.country)).to.equal("myanmar-burma"); - expect(ann.altitude).to.equal(43); +describe("parseAirportsIndex (offline)", function() { + it("lists every country with its slug", function() { + const countries = parseAirportsIndex(load("airports_index.html")); + expect(countries.map((c) => c.slug)).to.deep.equal(["brazil", "myanmar-burma", "uruguay"]); + expect(countries[1].name).to.equal("MYANMAR (BURMA)"); }); - it("matches a filter written with either spelling of the country", function() { - for (const slug of ["myanmar-burma", "Myanmar (Burma)"]) { - expect(parseAirportsJson(load("airports.json"), [slug]).length).to.equal(1); + it("produces slugs that match the Countries enum", function() { + const known = new Set(Object.values(Countries)); + + for (const country of parseAirportsIndex(load("airports_index.html"))) { + expect(known.has(country.slug), country.slug).to.equal(true); } }); - it("rejects numeric-looking junk instead of truncating it", function() { - const payload = JSON.stringify({ rows: [{ - name: "Junk Airport", iata: "JNK", icao: "JJNK", - lat: "43.30 N", lon: -8.37725, country: "Spain", alt: "-1", - }] }); - const [airport] = parseAirportsJson(payload); - - // One bad coordinate drops both; altitude is independent and survives. - expect(airport.latitude).to.equal(null); - expect(airport.longitude).to.equal(null); - expect(airport.altitude).to.equal(-1); + it("returns an empty list for an invalid payload", function() { + expect(parseAirportsIndex("")).to.deep.equal([]); + expect(parseAirportsIndex({ props: {} })).to.deep.equal([]); + expect(parseAirportsIndex({ props: { airportsByCountry: [1, "x", { name: null }] } })).to.deep.equal([]); }); +}); - it("agrees with the Python port on Unicode digits and stray control bytes", function() { - // Each port used to accept what the other rejected. - const read = (value) => { - const payload = JSON.stringify({ rows: [{ - name: "X", iata: "XXX", icao: "XXXX", country: "Spain", lat: value, lon: value, alt: value, - }] }); - return parseAirportsJson(payload)[0].altitude; - }; - expect(read("\u0664\u0663"), "Arabic-Indic digits").to.equal(null); - expect(read("\ufeff43"), "leading U+FEFF").to.equal(null); - expect(read("\u001c43"), "leading U+001C").to.equal(null); - expect(read(" 43 "), "plain spaces still trimmed").to.equal(43); - }); +describe("parseAirportsPage (offline)", function() { + it("extracts the airports of a country", function() { + const { airports, statePaths } = parseAirportsPage(loadJson("airports_country.json")); + expect(statePaths).to.deep.equal([]); + expect(airports.map((a) => a.iata)).to.deep.equal(["CYR", "MVD"]); - it("returns an empty list for an unknown country", function() { - expect(parseAirportsJson(load("airports.json"), ["atlantis"])).to.deep.equal([]); + const mvd = airports[1]; + expect(mvd.name).to.equal("Montevideo Carrasco International Airport"); + expect(mvd.icao).to.equal("SUMU"); + expect(mvd.country).to.equal("Uruguay"); }); - it("survives numbers written raw in the JSON that no double can hold", function() { - // Written as text: a JS literal is already rounded before it reaches here. - for (const literal of ["1".repeat(400), "9007199254740993", "-1".concat("1".repeat(399))]) { - const payload = `{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain","lat":${literal},"lon":1,"alt":2}]}`; - const [airport] = parseAirportsJson(payload); + it("carries no position: the listing has none", function() { + const { airports } = parseAirportsPage(loadJson("airports_country.json")); - expect(airport.latitude, literal.slice(0, 12)).to.satisfy( - (latitude) => latitude === null || Number.isFinite(latitude), - ); + for (const airport of airports) { + expect(airport.latitude).to.equal(null); + expect(airport.longitude).to.equal(null); + expect(airport.altitude).to.equal(null); } }); - it("keeps text fields as strings whatever the feed sends", function() { - // Anything else breaks getCountryFlag(airport.country). - for (const literal of ["null", "0", "false", "true", "[]", "{}", "123"]) { - const payload = `{"rows":[{"name":${literal},"iata":${literal},"icao":${literal},` + - `"country":${literal},"lat":1,"lon":2,"alt":3}]}`; - const [airport] = parseAirportsJson(payload); - - expect([airport.name, airport.iata, airport.icao, airport.country], literal) - .to.deep.equal(["", "", "", ""]); - } + it("returns the state pages of a split country", function() { + const { airports, statePaths } = parseAirportsPage(loadJson("airports_country_states.json")); + expect(airports).to.deep.equal([]); + expect(statePaths).to.deep.equal(["/data/airports/brazil/rj", "/data/airports/brazil/sp"]); }); - it("parses a body that arrived as raw bytes", function() { - // request() returns an ArrayBuffer on an unexpected content-type. - const json = load("airports.json"); + it("keeps the country name on a state page", function() { + const { airports, statePaths } = parseAirportsPage(loadJson("airports_state_sp.json")); + expect(statePaths).to.deep.equal([]); + expect(airports.map((a) => a.iata)).to.deep.equal(["GRU", "CGH"]); + expect(new Set(airports.map((a) => a.country))).to.deep.equal(new Set(["Brazil"])); + }); - for (const payload of [Buffer.from(json), new TextEncoder().encode(json), new TextEncoder().encode(json).buffer]) { - expect(parseAirportsJson(payload).length, payload.constructor.name).to.equal(airports.length); - } + it("reads an HTML page", function() { + const { airports } = parseAirportsPage(load("airports_country.html")); + expect(airports.map((a) => a.iata)).to.deep.equal(["RGN"]); + // FR24 spells it "Myanmar (Burma)" on the page and "myanmar-burma" in URLs. + expect(airports[0].country).to.equal("Myanmar (Burma)"); + expect(countryToSlug(airports[0].country)).to.equal(Countries.MYANMAR_BURMA); }); it("skips rows that are not objects, as the Python port does", function() { - const payload = "{\"rows\":[[1,2,3],\"x\",7,null," + - "{\"name\":\"Real\",\"iata\":\"RRR\",\"icao\":\"RRRR\",\"country\":\"Spain\",\"lat\":1,\"lon\":2,\"alt\":3}]}"; - - expect(parseAirportsJson(payload).map((airport) => airport.iata)).to.deep.equal(["RRR"]); + const payload = { props: { country: { name: "Nowhere" }, airports: [1, "x", null, [1, 2], { iata: "RRR" }] } }; + const { airports } = parseAirportsPage(payload); + expect(airports.map((a) => a.iata)).to.deep.equal(["RRR"]); }); - it("returns an empty list when the feed has no rows", function() { - expect(parseAirportsJson("")).to.deep.equal([]); - expect(parseAirportsJson("{}")).to.deep.equal([]); - expect(parseAirportsJson("not json")).to.deep.equal([]); + it("keeps text fields as strings whatever the page sends", function() { + // Anything else breaks getCountryFlag(airport.country). + const payload = { props: { country: { name: ["Brazil"] }, airports: [{ name: 42, iata: null, icao: { x: 1 } }] } }; + const [airport] = parseAirportsPage(payload).airports; + expect([airport.name, airport.iata, airport.icao, airport.country]).to.deep.equal(["", "", "", ""]); }); -}); + it("ignores malformed states", function() { + const payload = { props: { states: [{ url: "/data/airports/x/a" }, { url: "" }, { code: "B" }, "junk", { url: 3 }] } }; + expect(parseAirportsPage(payload)).to.deep.equal({ airports: [], statePaths: ["/data/airports/x/a"] }); + }); -describe("parseAirportsJson numeric coercion", function() { - for (const [label, input, expected] of NUMERIC_CASES) { - it(`reads ${label} as ${JSON.stringify(expected)}`, function() { - const payload = JSON.stringify({ rows: [{ - name: "X", iata: "XXX", icao: "XXXX", country: "Spain", - lat: input, lon: input, alt: input, - }] }); - const [airport] = parseAirportsJson(payload); - - expect(airport.latitude, "latitude").to.equal(expected); - expect(airport.longitude, "longitude").to.equal(expected); - expect(airport.altitude, "altitude").to.equal(expected); - }); - } + it("returns nothing for an invalid payload", function() { + const empty = { airports: [], statePaths: [] }; + expect(parseAirportsPage("")).to.deep.equal(empty); + expect(parseAirportsPage("not a data page")).to.deep.equal(empty); + expect(parseAirportsPage({ props: {} })).to.deep.equal(empty); + }); }); diff --git a/python/FlightRadarAPI/api.py b/python/FlightRadarAPI/api.py index e440e8a..d2d6646 100644 --- a/python/FlightRadarAPI/api.py +++ b/python/FlightRadarAPI/api.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import dataclasses +import logging import math from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict, Iterable, List, Optional, Tuple, Union @@ -11,9 +12,11 @@ from .entities.flight import Flight from .errors import AirportNotFoundError, LoginError from .flight_tracker_config import FlightTrackerConfig -from .parsers import country_to_slug, parse_airlines_html, parse_airports_json +from .parsers import country_to_slug, parse_airlines_html, parse_airports_index, parse_airports_page, parse_inertia_page from .request import APIClient, RetryPolicy +_logger = logging.getLogger(__name__) + # Some FR24 live-feed backends answer 200 with a well-formed envelope but no # flight entries -- indistinguishable from a legitimately empty result. The # "AWSALB" cookie then pins the session to that backend, so dropping it is what @@ -193,6 +196,12 @@ def get_airports( """ Return a list with all airports, optionally narrowed to some countries. + FR24 lists airports one page per country, and one per state for the + largest countries, so this fetches one page per country asked for (a few + hundred for every airport), `max_workers` at a time. The listing carries + the name, codes and country only: use get_airport(code) for the position + and altitude of an airport. + :param countries: Country names from the Countries enum, or their slug strings, as any iterable or a single value. Every country when omitted. """ @@ -207,13 +216,82 @@ def get_airports( if wanted is not None and len(wanted) == 0: return [] - response = self.__client.request( - Core.airports_json_url, headers=Core.json_headers, timeout=self.timeout, + # The index names every country, and declares the asset version the + # JSON requests below have to quote. + response = self.__client.request(Core.airports_data_url, headers=Core.html_headers, timeout=self.timeout) + index = parse_inertia_page(response.get_bytes_content()) + version = index.get("version") if index is not None else None + + if wanted is None: + slugs = [country["slug"] for country in parse_airports_index(index or {})] + else: + slugs = [country_to_slug(country) for country in wanted] + + slugs = list(dict.fromkeys(slug for slug in slugs if slug)) + version = version if isinstance(version, str) else None + + country_pages = self.__get_airport_pages( + [Core.airports_by_country_url.format(slug) for slug in slugs], version, ) - # get_content(), not get_json_content(): an html body reaches the parser's - # guard; a malformed JSON body still raises. The parser slugifies `wanted`. - return parse_airports_json(response.get_content(), wanted) + # Second pass for the countries split by state, every state at once. + state_owners = [index for index, (_, paths) in enumerate(country_pages) for _ in paths] + state_pages = self.__get_airport_pages( + [ + path if path.startswith("http") else Core.flightradar_base_url + path + for _, paths in country_pages for path in paths + ], + version, + ) + + # Grouped by country, in the order asked for, each country's states after it. + airports_by_country: List[List[Airport]] = [list(found) for found, _ in country_pages] + + for owner, (found, _) in zip(state_owners, state_pages): + airports_by_country[owner].extend(found) + + airports = [airport for found in airports_by_country for airport in found] + + # A country FR24 does not know answers with the index page, which lists + # neither airports nor states. + missing = [slug for slug, (found, paths) in zip(slugs, country_pages) if not found and not paths] + + if missing: + _logger.warning( + "get_airports: no airports found for %s — check the Countries enum.", ", ".join(missing), + ) + + return airports + + def __get_airport_pages(self, urls: List[str], version: Optional[str]) -> List[Tuple[List[Airport], List[str]]]: + """ + Fetch and parse the airport listing pages at `urls`, in order, max_workers at a time. + """ + if not urls: + return [] + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + return list(executor.map(lambda url: parse_airports_page(self.__get_inertia_page(url, version)), urls)) + + def __get_inertia_page(self, url: str, version: Optional[str]) -> Union[Dict, bytes]: + """ + Fetch a FR24 data page as its Inertia page object, falling back to the HTML. + + With the asset version FR24 answers the bare page JSON, a fraction of the + HTML. A 409 means the assets were redeployed since the version was read: + the HTML carries the same page object and needs no version. + """ + if version is not None: + headers = {**Core.inertia_headers, "x-inertia-version": version} + response = self.__client.request_standalone( + url, headers=headers, timeout=self.timeout, allowed_error_codes=[409], + ) + + if response.get_status_code() != 409: + return response.get_content() + + response = self.__client.request_standalone(url, headers=Core.html_headers, timeout=self.timeout) + return response.get_bytes_content() def get_bookmarks(self) -> Dict: """ diff --git a/python/FlightRadarAPI/core.py b/python/FlightRadarAPI/core.py index 5948e6c..93fcd5c 100644 --- a/python/FlightRadarAPI/core.py +++ b/python/FlightRadarAPI/core.py @@ -30,10 +30,12 @@ class Core: # Airports data URLs. api_airport_data_url = api_flightradar_base_url + "/airport.json" airport_data_url = flightradar_base_url + "/airports/traffic-stats/?airport={}" + # The listing is split by country, and by state for the largest countries. + # These are Inertia pages: the data rides JSON-encoded in the data-page + # attribute of the HTML, or comes as the whole body when the request carries + # `inertia_headers` and the asset version the index page declares. airports_data_url = flightradar_base_url + "/data/airports" - - # Every airport in one payload; the per-country pages above are client-rendered. - airports_json_url = flightradar_base_url + "/_json/airports.php" + airports_by_country_url = airports_data_url + "/{}" # Airlines data URL. airlines_data_url = flightradar_base_url + "/data/airlines" @@ -86,6 +88,12 @@ class Core: image_headers = headers.copy() image_headers["accept"] = "image/gif, image/jpg, image/jpeg, image/png" + # Asks an Inertia page for its page object instead of the HTML around it. + # `x-inertia-version` must match the deployed assets or FR24 answers 409. + inertia_headers = json_headers.copy() + inertia_headers["x-inertia"] = "true" + inertia_headers["x-requested-with"] = "XMLHttpRequest" + html_headers = { "accept": ( "text/html,application/xhtml+xml,application/xml;q=0.9," diff --git a/python/FlightRadarAPI/parsers.py b/python/FlightRadarAPI/parsers.py index 77667ad..59792b2 100644 --- a/python/FlightRadarAPI/parsers.py +++ b/python/FlightRadarAPI/parsers.py @@ -2,86 +2,111 @@ import json import logging -import math import re import unicodedata from enum import Enum -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from bs4 import BeautifulSoup +from bs4.element import Tag from .entities.airport import Airport _logger = logging.getLogger(__name__) -# ASCII only: \d would otherwise match Unicode digits such as "٤٣". -NUMERIC_PATTERN = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$", re.ASCII) +# An airline code badge: a two-letter IATA code or a three-letter ICAO code. +# ASCII only, so a stray Unicode letter does not pass for a code. +AIRLINE_CODE_PATTERN = re.compile(r"^[A-Z0-9]{2,3}$", re.ASCII) -# str.strip() and trim() disagree on U+001C-U+001F and U+FEFF: pick one set. -SURROUNDING_SPACE = " \t\n\r\f\v" +# A fleet size as the page prints it: "340" or "3,878". +AIRCRAFT_COUNT_PATTERN = re.compile(r"^\d{1,3}(?:[,.]\d{3})*$|^\d+$", re.ASCII) -# Number.MAX_SAFE_INTEGER, the largest integer JavaScript holds exactly. -MAX_EXACT_INTEGER = 2 ** 53 - 1 +AIRLINES_PATH_PREFIX = "/data/airlines/" + + +def _is_airline_link(href: object) -> bool: + return isinstance(href, str) and href.startswith(AIRLINES_PATH_PREFIX) + + +def _leaf_texts(root: Tag, tag: str) -> List[str]: + """ + Text of the `tag` elements under `root` that hold no other element. + + The name element is left out by its `title` attribute: an airline called + "TAP" would otherwise read as an ICAO code. + """ + return [ + element.get_text(strip=True) + for element in root.find_all(tag) + if element.find() is None and not element.has_attr("title") + ] + + +def _airline_name(link: Tag) -> str: + """ + The airline name of a listing entry. + + The name sits in the element titled with it, with the logo's aria-label as a + fallback; the anchor text also carries the codes and the fleet size. + """ + for attribute in ("title", "aria-label"): + element = link.find(lambda tag, name=attribute: tag.has_attr(name)) + + if isinstance(element, Tag): + return str(element[attribute]).strip() + + return link.get_text(strip=True) def parse_airlines_html(html: bytes) -> List[Dict]: """ - Parse the airlines listing HTML page into a list of airline dicts. + Parse the airlines listing page into a list of airline dicts. + + Every airline is a link to its data page, with the IATA and ICAO codes as + badges next to the name. The "Largest fleets" section repeats a few airlines + with their aircraft count; an airline the page lists twice becomes one entry + carrying everything the page said about it. """ soup = BeautifulSoup(html, "html.parser") - tbody = soup.find("tbody") + links = soup.find_all("a", href=_is_airline_link) - if not tbody: + if not links: _logger.warning( - "parse_airlines_html: no in response — FR24 page layout may have changed." + "parse_airlines_html: no airline links in response — FR24 page layout may have changed." ) return [] - airlines = [] - - for tr in tbody.find_all("tr"): - td_notranslate = tr.find("td", class_="notranslate") + airlines: Dict[str, Dict] = {} - if not td_notranslate: - continue - - a_element = td_notranslate.find("a", href=lambda href: href and href.startswith("/data/airlines")) + for link in links: + name = _airline_name(link) - if not a_element: + if len(name) < 2: continue - airline_name = a_element.get_text(strip=True) + codes = [text for text in _leaf_texts(link, "span") if AIRLINE_CODE_PATTERN.match(text)] + iata = next((code for code in codes if len(code) == 2), None) + icao = next((code for code in codes if len(code) == 3), None) - if len(airline_name) < 2: + # Section links ("All") point under /data/airlines/ too; a code tells + # an airline apart from them. + if iata is None and icao is None: continue - td_elements = tr.find_all("td") - iata = None - icao = None - - if len(td_elements) >= 4: - codes_text = td_elements[3].get_text(strip=True) - - if " / " in codes_text: - parts = codes_text.split(" / ") - if len(parts) == 2: - iata = parts[0].strip() - icao = parts[1].strip() - elif len(codes_text) == 2: - iata = codes_text - elif len(codes_text) == 3: - icao = codes_text - - n_aircrafts = None + n_aircrafts = next( + (int(re.sub(r"\D", "", text)) for text in _leaf_texts(link, "strong") if AIRCRAFT_COUNT_PATTERN.match(text)), + None, + ) - if len(td_elements) >= 5: - aircrafts_text = td_elements[4].get_text(strip=True) - if aircrafts_text: - n_aircrafts = int(aircrafts_text.split(" ", maxsplit=1)[0].strip()) + airline = airlines.setdefault( + str(link["href"]), {"Name": name, "ICAO": None, "IATA": None, "n_aircrafts": None}, + ) - airlines.append({"Name": airline_name, "ICAO": icao, "IATA": iata, "n_aircrafts": n_aircrafts}) + for key, value in (("ICAO", icao), ("IATA", iata), ("n_aircrafts", n_aircrafts)): + if airline[key] is None: + airline[key] = value - return airlines + return list(airlines.values()) def country_to_slug(country: object) -> str: @@ -115,125 +140,103 @@ def _to_text(value: object) -> str: return value if isinstance(value, str) else "" -def _to_number(value: object) -> Optional[Union[int, float]]: +def parse_inertia_page(payload: Union[bytes, str, Dict]) -> Optional[Dict]: """ - Coerce a numeric feed field into a number, or None when it is unusable. - - An unusable coordinate must not become 0.0: that would place the airport in - the Gulf of Guinea instead of marking its position as unknown. + Return the Inertia page object (component, props, version) of a FR24 data page. - Plain whole numbers stay ints, so an altitude reads 2436 in both ports rather - than 2436.0 here -- the feed sends `alt` as an int for most rows and as a - string ("-1") for the rest. Anything written with a decimal point or an - exponent stays a float: "2436.0" and "1e3" are floats here and integral - Numbers in Node, equal in value but not in type. + Served as HTML, the page keeps it JSON-encoded in the data-page attribute of + its root element; asked with Core.inertia_headers, FR24 sends it as the body, + which `request.get_content()` has already decoded into a dict. """ - # bool is an int here, and str([43]) would invent a number. - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - - if isinstance(value, str): - stripped = value.strip(SURROUNDING_SPACE) + if isinstance(payload, dict): + page: object = payload + else: + root = BeautifulSoup(payload, "html.parser").find(lambda tag: tag.has_attr("data-page")) - if not NUMERIC_PATTERN.match(stripped): + if not isinstance(root, Tag): + _logger.warning("parse_inertia_page: no data-page attribute in response — FR24 page layout may have changed.") return None - text: Union[int, float, str] = stripped - else: - text = value - - # One conversion, so overflow is handled once: Python ints have no ceiling. - try: - number = float(text) - except OverflowError: - return None + try: + page = json.loads(str(root["data-page"])) + except ValueError: + _logger.warning("parse_inertia_page: data-page is not valid JSON — FR24 page layout may have changed.") + return None - if not math.isfinite(number): + if not isinstance(page, dict) or not isinstance(page.get("props"), dict): + _logger.warning("parse_inertia_page: no page props in response — FR24 page layout may have changed.") return None - try: - exact = int(text) - except (ValueError, OverflowError): - return number - - # int only when lossless (so -23.4 survives) and only where JavaScript is exact. - return exact if exact == number and abs(exact) <= MAX_EXACT_INTEGER else number + return page -def parse_airports_json( - payload: Union[bytes, str, Dict], countries: Optional[Iterable[object]] = None, -) -> List[Airport]: +def parse_airports_index(payload: Union[bytes, str, Dict]) -> List[Dict]: """ - Parse the airports JSON feed into a list of Airport instances. + Parse the airports index (Core.airports_data_url) into its countries. - :param payload: Body of Core.airports_json_url. - :param countries: Country slugs, or Countries members. Every airport is kept when omitted. + Each entry has the country "name" as FR24 prints it ("UNITED STATES") and the + "slug" of its listing page, which is also its Countries enum value. """ - if isinstance(payload, (bytes, str)): - try: - data = json.loads(payload) - except ValueError: - _logger.warning("parse_airports_json: response is not valid JSON — FR24 feed may have changed.") - return [] - else: - data = payload - - rows = data.get("rows") if isinstance(data, dict) else None + page = parse_inertia_page(payload) + rows = page["props"].get("airportsByCountry") if page is not None else None if not isinstance(rows, list): - _logger.warning('parse_airports_json: no "rows" array in response — FR24 feed may have changed.') + _logger.warning('parse_airports_index: no "airportsByCountry" list in response — FR24 page layout may have changed.') return [] - wanted = {country_to_slug(country) for country in countries} if countries is not None else None - matched = set() - unpositioned = [] - airports = [] + countries = [] for row in rows: if not isinstance(row, dict): continue - # Slugified only when filtering: it is otherwise unused, and this loop - # runs over every airport in the feed. - if wanted is not None: - slug = country_to_slug(row.get("country")) + name = _to_text(row.get("name")) + slug = country_to_slug(name) - if slug not in wanted: - continue - matched.add(slug) + if slug: + countries.append({"name": name, "slug": slug}) - latitude = _to_number(row.get("lat")) - longitude = _to_number(row.get("lon")) + return countries - # One bad coordinate drops both: half a position reads as located. - if latitude is None or longitude is None: - latitude = longitude = None - unpositioned.append(_to_text(row.get("name"))) - airports.append(Airport(basic_info={ - "name": _to_text(row.get("name")), - "icao": _to_text(row.get("icao")), - "iata": _to_text(row.get("iata")), - "lat": latitude, - "lon": longitude, - "alt": _to_number(row.get("alt")), - "country": _to_text(row.get("country")), - })) - - # One line, not one per row: the feed carries every airport. - if unpositioned: - _logger.warning( - "parse_airports_json: %d airport(s) had unusable coordinates and carry no position (e.g. %s).", - len(unpositioned), ", ".join(repr(name) for name in unpositioned[:3]), - ) +def parse_airports_page(payload: Union[bytes, str, Dict]) -> Tuple[List[Airport], List[str]]: + """ + Parse a country or state airports page into (airports, state_paths). - if wanted is not None: - missing = sorted(wanted - matched) + A country with many airports lists its states instead, each with the path of + its own page; those paths come back for the caller to fetch. The listing has + no coordinates or altitude: `get_airport(code)` has them for one airport. + """ + page = parse_inertia_page(payload) + + if page is None: + return [], [] + + props = page["props"] + country = props.get("country") + country_name = _to_text(country.get("name")) if isinstance(country, dict) else "" - if missing: - _logger.warning( - "parse_airports_json: no airports found for %s — check the Countries enum.", - ", ".join(missing), - ) + rows = props.get("airports") + states = props.get("states") - return airports + airports = [ + Airport(basic_info={ + "name": _to_text(row.get("name")), + "icao": _to_text(row.get("icao")), + "iata": _to_text(row.get("iata")), + "lat": None, + "lon": None, + "alt": None, + "country": country_name, + }) + for row in (rows if isinstance(rows, list) else []) + if isinstance(row, dict) + ] + + state_paths = [ + state["url"] + for state in (states if isinstance(states, list) else []) + if isinstance(state, dict) and isinstance(state.get("url"), str) and state["url"] + ] + + return airports, state_paths diff --git a/python/tests/fixtures/airlines.html b/python/tests/fixtures/airlines.html index 3c56b99..020dfd5 100644 --- a/python/tests/fixtures/airlines.html +++ b/python/tests/fixtures/airlines.html @@ -1,60 +1,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- LATAM Airlines - xxLA / LAN340 aircraft
- Gol - xxG3 / GLO140 aircraft
- Delta Air Lines - xxDL / DAL900 aircraft
- Sky2 - xxSK10 aircraft
- SkyTeam - xxSKT15 aircraft
placeholder
- Should Skip -
- + + +Airlines - Flightradar24 + +
+
+

Largest fleets

+ All +
+ +
+
+

All airlines

+ +
+ + diff --git a/python/tests/fixtures/airports.json b/python/tests/fixtures/airports.json deleted file mode 100644 index 924f9db..0000000 --- a/python/tests/fixtures/airports.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1786010717", - "rows": [ - { - "name": "Sao Paulo Guarulhos International Airport", - "iata": "GRU", - "icao": "SBGR", - "lat": -23.429991, - "lon": -46.4674, - "country": "Brazil", - "alt": 2436 - }, - { - "name": "Rio de Janeiro Galeao International Airport", - "iata": "GIG", - "icao": "SBGL", - "lat": -22.805696, - "lon": -43.25523, - "country": "Brazil", - "alt": 22 - }, - { - "name": "Sao Paulo Congonhas Airport", - "iata": "CGH", - "icao": "SBSP", - "lat": -23.627348, - "lon": -46.655987, - "country": "Brazil", - "alt": 2614 - }, - { - "name": "Los Angeles International Airport", - "iata": "LAX", - "icao": "KLAX", - "lat": 33.94252, - "lon": -118.406998, - "country": "United States", - "alt": 125 - }, - { - "name": "New York John F. Kennedy International Airport", - "iata": "JFK", - "icao": "KJFK", - "lat": 40.639751, - "lon": -73.7789, - "country": "United States", - "alt": 14 - }, - { - "name": "A Coruna Airport", - "iata": "LCG", - "icao": "LECO", - "lat": 43.302059, - "lon": -8.37725, - "country": "Spain", - "alt": 326 - }, - { - "name": "Ann Airport", - "iata": "VBA", - "icao": "VYAN", - "lat": 19.770643, - "lon": 94.026405, - "country": "Myanmar (Burma)", - "alt": "43" - }, - { - "name": "Broken Coords Airport", - "iata": "BAD", - "icao": "SBAD", - "lat": "n/a", - "lon": "", - "country": "Brazil", - "alt": 0 - } - ] -} diff --git a/python/tests/fixtures/airports_country.html b/python/tests/fixtures/airports_country.html new file mode 100644 index 0000000..833d4f4 --- /dev/null +++ b/python/tests/fixtures/airports_country.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/python/tests/fixtures/airports_country.json b/python/tests/fixtures/airports_country.json new file mode 100644 index 0000000..09b4811 --- /dev/null +++ b/python/tests/fixtures/airports_country.json @@ -0,0 +1,51 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "uruguay" + }, + "query": [] + }, + "country": { + "id": 238, + "name": "Uruguay", + "slug": "uruguay" + }, + "airports": [ + { + "id": 787, + "name": "Colonia Laguna de Los Patos International Airport", + "city": "Colonia", + "iata": "CYR", + "icao": "SUCA", + "total": 8 + }, + { + "id": 2349, + "name": "Montevideo Carrasco International Airport", + "city": "Montevideo", + "iata": "MVD", + "icao": "SUMU", + "total": 632 + } + ], + "airportImages": { + "SUMU": { + "thumbnail": "https://cdn.jetphotos.com/200/5/1.jpg", + "copyright": "Someone" + } + }, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/uruguay", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/fixtures/airports_country_states.json b/python/tests/fixtures/airports_country_states.json new file mode 100644 index 0000000..ff8bca1 --- /dev/null +++ b/python/tests/fixtures/airports_country_states.json @@ -0,0 +1,41 @@ +{ + "component": "Data/AirportsByCountry", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-country", + "params": { + "country": "brazil" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "states": [ + { + "code": "RJ", + "name": "Rio de Janeiro", + "total": 1, + "url": "/data/airports/brazil/rj" + }, + { + "code": "SP", + "name": "Sao Paulo", + "total": 2, + "url": "/data/airports/brazil/sp" + } + ], + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/fixtures/airports_index.html b/python/tests/fixtures/airports_index.html new file mode 100644 index 0000000..a55ed2f --- /dev/null +++ b/python/tests/fixtures/airports_index.html @@ -0,0 +1,8 @@ + + +Flightradar24 + +
+ + + diff --git a/python/tests/fixtures/airports_state_rj.json b/python/tests/fixtures/airports_state_rj.json new file mode 100644 index 0000000..77d7a6b --- /dev/null +++ b/python/tests/fixtures/airports_state_rj.json @@ -0,0 +1,44 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "rj" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "RJ", + "name": "Rio de Janeiro", + "slug": "rj" + }, + "airports": [ + { + "id": 1252, + "name": "Rio de Janeiro Galeao International Airport", + "city": "Rio de Janeiro", + "iata": "GIG", + "icao": "SBGL", + "total": 2500 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/rj", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/fixtures/airports_state_sp.json b/python/tests/fixtures/airports_state_sp.json new file mode 100644 index 0000000..46c60c6 --- /dev/null +++ b/python/tests/fixtures/airports_state_sp.json @@ -0,0 +1,52 @@ +{ + "component": "Data/AirportsByState", + "props": { + "errors": {}, + "name": "Flightradar24 API", + "host": "www.flightradar24.com", + "locale": "en", + "route": { + "name": "web.data.airports.by-state", + "params": { + "country": "brazil", + "state": "sp" + }, + "query": [] + }, + "country": { + "id": 32, + "name": "Brazil", + "slug": "brazil" + }, + "state": { + "code": "SP", + "name": "Sao Paulo", + "slug": "sp" + }, + "airports": [ + { + "id": 1255, + "name": "Sao Paulo Guarulhos International Airport", + "city": "Sao Paulo", + "iata": "GRU", + "icao": "SBGR", + "total": 6796 + }, + { + "id": 1254, + "name": "Sao Paulo Congonhas Airport", + "city": "Sao Paulo", + "iata": "CGH", + "icao": "SBSP", + "total": 3100 + } + ], + "airportImages": {}, + "busiestAirports": [], + "bestAirports": [] + }, + "url": "/data/airports/brazil/sp", + "version": "919b94d2a33f365a34f99828f6eeec6a", + "clearHistory": false, + "encryptHistory": false +} diff --git a/python/tests/test_airports_offline.py b/python/tests/test_airports_offline.py new file mode 100644 index 0000000..39588f0 --- /dev/null +++ b/python/tests/test_airports_offline.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +"""Offline tests for the page crawl behind ``get_airports()``. + +FR24 lists airports one Inertia page per country, and per state for the largest +countries. These tests pin down which pages the crawl asks for and how it reads +them, against the bundled fixtures, so the PR gate covers the orchestration +without reaching FR24. +""" + +import json +import os +from typing import Any, Dict, List, Optional, Union + +from FlightRadarAPI import Countries, FlightRadar24API +from FlightRadarAPI.core import Core + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") + + +def _load(name: str) -> bytes: + with open(os.path.join(FIXTURES, name), "rb") as f: + return f.read() + + +# Every page the fixtures model, by the path FR24 serves it at. +PAGES: Dict[str, str] = { + "/data/airports": "airports_index.html", + "/data/airports/uruguay": "airports_country.json", + "/data/airports/brazil": "airports_country_states.json", + "/data/airports/brazil/sp": "airports_state_sp.json", + "/data/airports/brazil/rj": "airports_state_rj.json", + "/data/airports/myanmar-burma": "airports_country.html", +} + + +class _FakeResponse: + def __init__(self, status_code: int, content: Union[Dict[str, Any], bytes]) -> None: + self._status_code = status_code + self._content = content + + def get_status_code(self) -> int: + return self._status_code + + def get_content(self) -> Union[Dict[str, Any], bytes]: + return self._content + + def get_bytes_content(self) -> bytes: + if not isinstance(self._content, bytes): + raise ValueError("Expected bytes response, got JSON") + return self._content + + +class _FakeClient: + """Serves the fixture pages the way FR24 does. + + A request carrying the inertia headers gets the page JSON when the version + matches and a 409 otherwise; a plain request gets the HTML. The index page + only exists as HTML, like the real one as the crawl fetches it. + """ + + def __init__(self, version: Optional[str] = "919b94d2a33f365a34f99828f6eeec6a") -> None: + self.version = version + self.calls: List[Dict[str, Any]] = [] + + def _serve(self, url: str, **kwargs: Any) -> _FakeResponse: + self.calls.append({"url": url, **kwargs}) + path = url.replace(Core.flightradar_base_url, "") + fixture = PAGES.get(path, "airports_index.html") + headers = kwargs.get("headers") or {} + + if headers.get("x-inertia") == "true": + if headers.get("x-inertia-version") != self.version: + assert 409 in (kwargs.get("allowed_error_codes") or []), "a 409 must not raise" + return _FakeResponse(409, b"") + page = json.loads(_load(fixture)) if fixture.endswith(".json") else _page_of(_load(fixture)) + return _FakeResponse(200, page) + + if fixture.endswith(".json"): + fixture = fixture.replace(".json", ".html") + if not os.path.exists(os.path.join(FIXTURES, fixture)): + fixture = "airports_country.html" + return _FakeResponse(200, _load(fixture)) + + request = _serve + request_standalone = _serve + + +def _page_of(html: bytes) -> Dict[str, Any]: + from FlightRadarAPI.parsers import parse_inertia_page + page = parse_inertia_page(html) + assert page is not None + return page + + +def _api(client: _FakeClient) -> FlightRadar24API: + api = FlightRadar24API() + # Name-mangled private attribute -- set it the way Python stores it. + api._FlightRadar24API__client = client # type: ignore[attr-defined] + return api + + +def _paths(client: _FakeClient) -> List[str]: + return [call["url"].replace(Core.flightradar_base_url, "") for call in client.calls] + + +def test_get_airports_crawls_the_country_then_its_states(): + client = _FakeClient() + + airports = _api(client).get_airports([Countries.BRAZIL]) + + assert [a.iata for a in airports] == ["GIG", "GRU", "CGH"] + assert {a.country for a in airports} == {"Brazil"} + assert _paths(client) == ["/data/airports", "/data/airports/brazil", "/data/airports/brazil/rj", "/data/airports/brazil/sp"] + + +def test_get_airports_asks_for_the_page_json_with_the_index_version(): + client = _FakeClient() + + _api(client).get_airports(Countries.URUGUAY) + + index, country = client.calls + assert "x-inertia" not in index["headers"] + assert country["headers"]["x-inertia"] == "true" + assert country["headers"]["x-inertia-version"] == "919b94d2a33f365a34f99828f6eeec6a" + + +def test_get_airports_falls_back_to_html_when_the_assets_were_redeployed(): + client = _FakeClient(version="something-newer") + + airports = _api(client).get_airports(["uruguay"]) + + # The index answered with a stale version, the JSON request got a 409 and + # the HTML page was read instead. + assert len(airports) == 1 + assert _paths(client) == ["/data/airports", "/data/airports/uruguay", "/data/airports/uruguay"] + assert client.calls[2]["headers"] == Core.html_headers + + +def test_get_airports_without_countries_walks_the_whole_index(): + client = _FakeClient() + + airports = _api(client).get_airports() + + assert [a.iata for a in airports] == ["GIG", "GRU", "CGH", "RGN", "CYR", "MVD"] + assert _paths(client) == [ + "/data/airports", + "/data/airports/brazil", "/data/airports/myanmar-burma", "/data/airports/uruguay", + "/data/airports/brazil/rj", "/data/airports/brazil/sp", + ] + + +def test_get_airports_accepts_mixed_spellings_and_drops_repeats(): + client = _FakeClient() + + airports = _api(client).get_airports(["Uruguay", Countries.URUGUAY, "uruguay"]) + + assert [a.iata for a in airports] == ["CYR", "MVD"] + assert _paths(client) == ["/data/airports", "/data/airports/uruguay"] + + +def test_get_airports_with_an_empty_filter_makes_no_request(): + client = _FakeClient() + + assert _api(client).get_airports([]) == [] + assert _api(client).get_airports(iter([])) == [] + assert client.calls == [] + + +def test_get_airports_unknown_country_returns_nothing_and_warns(caplog): + client = _FakeClient() + + with caplog.at_level("WARNING", logger="FlightRadarAPI.api"): + airports = _api(client).get_airports(["atlantis"]) + + assert airports == [] + assert "no airports found for atlantis" in caplog.text diff --git a/python/tests/test_parsers_offline.py b/python/tests/test_parsers_offline.py index cf675ac..133b62b 100644 --- a/python/tests/test_parsers_offline.py +++ b/python/tests/test_parsers_offline.py @@ -9,28 +9,15 @@ """ import json -import math import os -import pytest - from FlightRadarAPI.core import Countries -from FlightRadarAPI.parsers import country_to_slug, parse_airlines_html, parse_airports_json +from FlightRadarAPI.parsers import ( + country_to_slug, parse_airlines_html, parse_airports_index, parse_airports_page, parse_inertia_page, +) FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") -# Keep in step with the same list in nodejs/tests/testParsersOffline.js. -NUMERIC_CASES = [ - ("whitespace", " ", None), - ("an array holding a number", [43], None), - ("a coordinate with a hemisphere suffix", "43.30 N", None), - ("an exponent overflowing a double", "1e999", None), - ("400 plain digits", "1" * 400, None), - ("a whole number as a string", "2436", 2436), - ("a negative decimal", "-23.4", -23.4), - ("a genuine zero", 0, 0), -] - def _load(name: str) -> bytes: with open(os.path.join(FIXTURES, name), "rb") as f: @@ -68,187 +55,135 @@ def test_parse_airlines_html_parses_aircraft_count(): by_name = {a["Name"]: a for a in airlines} assert by_name["LATAM Airlines"]["n_aircrafts"] == 340 assert by_name["Gol"]["n_aircrafts"] == 140 + # Thousands separator, as the page prints large fleets. + assert by_name["Delta Air Lines"]["n_aircrafts"] == 1900 + # Only the "Largest fleets" section carries a count. + assert by_name["Sky2"]["n_aircrafts"] is None -def test_parse_airlines_html_skips_invalid_rows(): +def test_parse_airlines_html_merges_an_airline_listed_in_two_sections(): airlines = parse_airlines_html(_load("airlines.html")) - # 5 valid rows; 2 invalid (no notranslate, wrong href) must be skipped. - assert len(airlines) == 5 - - -def test_parse_airlines_html_empty_input_returns_empty_list(): - assert parse_airlines_html(b"") == [] - assert parse_airlines_html(b"

no tbody here

") == [] - -# --- parse_airports_json --- - - -def test_parse_airports_json_extracts_basic_fields(): - airports = parse_airports_json(_load("airports.json")) - by_iata = {a.iata: a for a in airports} - assert "GRU" in by_iata and "GIG" in by_iata - - gru = by_iata["GRU"] - assert gru.icao == "SBGR" - assert gru.country == "Brazil" - assert abs(gru.latitude - (-23.429991)) < 1e-6 - assert abs(gru.longitude - (-46.4674)) < 1e-6 - assert gru.altitude == 2436 + # Delta and Gol appear under "Largest fleets" and again under "All airlines". + assert [a["Name"] for a in airlines].count("Delta Air Lines") == 1 + assert [a["Name"] for a in airlines].count("Gol") == 1 -def test_parse_airports_json_keeps_every_country_without_filter(): - countries = {a.country for a in parse_airports_json(_load("airports.json"))} - assert {"Brazil", "United States", "Spain"} <= countries - - -def test_parse_airports_json_filters_by_country_slug(): - airports = parse_airports_json(_load("airports.json"), ["united-states"]) - assert airports - assert all(a.country == "United States" for a in airports) - - -def test_parse_airports_json_accepts_several_countries(): - airports = parse_airports_json(_load("airports.json"), ["brazil", "spain"]) - assert {a.country for a in airports} == {"Brazil", "Spain"} +def test_parse_airlines_html_keeps_a_code_shaped_name(): + airlines = parse_airlines_html(_load("airlines.html")) + by_name = {a["Name"]: a for a in airlines} + assert by_name["TAP"] == {"Name": "TAP", "ICAO": "TAP", "IATA": "TP", "n_aircrafts": None} -def test_parse_airports_json_invalid_coordinates_become_none(): - """Regression: invalid coords used to be silently coerced to (0.0, 0.0) - placing the airport in the Gulf of Guinea. They must be None now.""" - airports = parse_airports_json(_load("airports.json")) - bad = next(a for a in airports if a.iata == "BAD") - assert bad.latitude is None - assert bad.longitude is None +def test_parse_airlines_html_skips_invalid_rows(): + airlines = parse_airlines_html(_load("airlines.html")) + # 6 airlines; the section link, the entry with no link, the wrong href and + # the one-letter name must be skipped. + assert len(airlines) == 6 + assert "All" not in [a["Name"] for a in airlines] -def test_parse_airports_json_rejects_numeric_looking_junk(): - payload = json.dumps({"rows": [{ - "name": "Junk Airport", "iata": "JNK", "icao": "JJNK", - "lat": "43.30 N", "lon": -8.37725, "country": "Spain", "alt": "-1", - }]}) - airport = parse_airports_json(payload)[0] +def test_parse_airlines_html_empty_input_returns_empty_list(): + assert parse_airlines_html(b"") == [] + assert parse_airlines_html(b"

no airlines here

") == [] - # One bad coordinate drops both; altitude is independent and survives. - assert airport.latitude is None - assert airport.longitude is None - assert airport.altitude == -1 - assert isinstance(airport.altitude, int) +# --- parse_inertia_page --- -@pytest.mark.parametrize("value,expected", [ - ("٤٣", None), # Arabic-Indic digits, which \d matched before re.ASCII - ("43", None), # U+FEFF, which String.trim() drops and str.strip() keeps - ("43", None), # U+001C, which str.strip() drops and String.trim() keeps - (" 43 ", 43), # plain spaces are still trimmed -]) -def test_parse_airports_json_agrees_with_node_on_unicode(value, expected): - payload = json.dumps({"rows": [{ - "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", - "lat": value, "lon": value, "alt": value, - }]}) +def test_parse_inertia_page_reads_the_data_page_attribute(): + page = parse_inertia_page(_load("airports_index.html")) + assert page["component"] == "Data/Airports" + assert page["version"] == "919b94d2a33f365a34f99828f6eeec6a" + assert "airportsByCountry" in page["props"] - assert parse_airports_json(payload)[0].altitude == expected +def test_parse_inertia_page_accepts_a_decoded_json_body(): + body = json.loads(_load("airports_country.json")) + assert parse_inertia_page(body) is body -@pytest.mark.parametrize( - "value,expected", - [(case[1], case[2]) for case in NUMERIC_CASES], - ids=[case[0] for case in NUMERIC_CASES], -) -def test_parse_airports_json_numeric_coercion(value, expected): - payload = json.dumps({"rows": [{ - "name": "X", "iata": "XXX", "icao": "XXXX", "country": "Spain", - "lat": value, "lon": value, "alt": value, - }]}) - airport = parse_airports_json(payload)[0] - assert airport.latitude == expected - assert airport.longitude == expected - assert airport.altitude == expected +def test_parse_inertia_page_rejects_unrecognised_payloads(): + assert parse_inertia_page(b"") is None + assert parse_inertia_page(b"
") is None + assert parse_inertia_page(b"
") is None + assert parse_inertia_page({"component": "Data/Airports"}) is None -@pytest.mark.parametrize("literal", ["1" * 400, "9007199254740993", "-" + "1" * 400]) -def test_parse_airports_json_survives_raw_numbers_no_double_can_hold(literal): - """Written as text: json.loads turns these into unbounded ints.""" - payload = ( - '{"rows":[{"name":"X","iata":"XXX","icao":"XXXX","country":"Spain",' - f'"lat":{literal},"lon":1,"alt":2}}]}}' - ) - latitude = parse_airports_json(payload)[0].latitude +# --- parse_airports_index --- - assert latitude is None or math.isfinite(latitude) +def test_parse_airports_index_lists_every_country_with_its_slug(): + countries = parse_airports_index(_load("airports_index.html")) + assert [c["slug"] for c in countries] == ["brazil", "myanmar-burma", "uruguay"] + assert countries[1]["name"] == "MYANMAR (BURMA)" -@pytest.mark.parametrize("literal", ["null", "0", "false", "true", "[]", "{}", "123"]) -def test_parse_airports_json_keeps_text_fields_as_strings(literal): - """Anything else breaks get_country_flag(airport.country).""" - payload = ( - '{"rows":[{"name":%s,"iata":%s,"icao":%s,"country":%s,"lat":1,"lon":2,"alt":3}]}' - % (literal, literal, literal, literal) - ) - airport = parse_airports_json(payload)[0] +def test_parse_airports_index_slugs_match_the_countries_enum(): + slugs = {c["slug"] for c in parse_airports_index(_load("airports_index.html"))} + assert slugs <= {c.value for c in Countries} - assert [airport.name, airport.iata, airport.icao, airport.country] == ["", "", "", ""] +def test_parse_airports_index_invalid_payload_returns_empty_list(): + assert parse_airports_index(b"") == [] + assert parse_airports_index({"props": {}}) == [] + assert parse_airports_index({"props": {"airportsByCountry": [1, "x", {"name": None}]}}) == [] -def test_parse_airports_json_country_spelling_and_slug_round_trip(): - ann = next(a for a in parse_airports_json(_load("airports.json")) if a.iata == "VBA") - assert ann.country == "Myanmar (Burma)" - assert country_to_slug(ann.country) == "myanmar-burma" - assert ann.altitude == 43 +# --- parse_airports_page --- +def test_parse_airports_page_extracts_the_airports_of_a_country(): + airports, state_paths = parse_airports_page(json.loads(_load("airports_country.json"))) + assert state_paths == [] + assert [a.iata for a in airports] == ["CYR", "MVD"] -def test_parse_airports_json_filter_accepts_either_country_spelling(): - for slug in ["myanmar-burma", "Myanmar (Burma)"]: - assert len(parse_airports_json(_load("airports.json"), [slug])) == 1 + mvd = airports[1] + assert mvd.name == "Montevideo Carrasco International Airport" + assert mvd.icao == "SUMU" + assert mvd.country == "Uruguay" -def test_parse_airports_json_keeps_altitude_as_int(): - """Both ports must agree on altitude: 2436, never 2436.0.""" - gru = next(a for a in parse_airports_json(_load("airports.json")) if a.iata == "GRU") - assert isinstance(gru.altitude, int) +def test_parse_airports_page_listing_carries_no_position(): + airports, _ = parse_airports_page(json.loads(_load("airports_country.json"))) + assert all(a.latitude is None and a.longitude is None and a.altitude is None for a in airports) -def test_parse_airports_json_skips_rows_that_are_not_objects(): - payload = ( - '{"rows":[[1,2,3],"x",7,null,' - '{"name":"Real","iata":"RRR","icao":"RRRR","country":"Spain","lat":1,"lon":2,"alt":3}]}' - ) +def test_parse_airports_page_returns_the_state_pages_of_a_split_country(): + airports, state_paths = parse_airports_page(json.loads(_load("airports_country_states.json"))) + assert airports == [] + assert state_paths == ["/data/airports/brazil/rj", "/data/airports/brazil/sp"] - assert [a.iata for a in parse_airports_json(payload)] == ["RRR"] +def test_parse_airports_page_state_page_keeps_the_country_name(): + airports, state_paths = parse_airports_page(json.loads(_load("airports_state_sp.json"))) + assert state_paths == [] + assert [a.iata for a in airports] == ["GRU", "CGH"] + assert {a.country for a in airports} == {"Brazil"} -def test_parse_airports_json_unknown_country_returns_empty_list(): - assert parse_airports_json(_load("airports.json"), ["atlantis"]) == [] +def test_parse_airports_page_reads_an_html_page(): + airports, _ = parse_airports_page(_load("airports_country.html")) + assert [a.iata for a in airports] == ["RGN"] + # FR24 spells it "Myanmar (Burma)" on the page and "myanmar-burma" in URLs. + assert airports[0].country == "Myanmar (Burma)" + assert country_to_slug(airports[0].country) == Countries.MYANMAR_BURMA.value -def test_parse_airports_json_invalid_payload_returns_empty_list(): - assert parse_airports_json(b"") == [] - assert parse_airports_json(b"{}") == [] - assert parse_airports_json(b"not json") == [] +def test_parse_airports_page_skips_rows_that_are_not_objects(): + payload = {"props": {"country": {"name": "Nowhere"}, "airports": [1, "x", None, {"iata": "RRR"}]}} + airports, _ = parse_airports_page(payload) + assert [a.iata for a in airports] == ["RRR"] -# --- country_to_slug --- -def test_country_to_slug_matches_countries_enum_spelling(): - assert country_to_slug("United States") == "united-states" - assert country_to_slug("Democratic Republic Of The Congo") == "democratic-republic-of-the-congo" - assert country_to_slug("Curacao") == "curacao" - assert country_to_slug("Curaçao") == "curacao" - assert country_to_slug(None) == "" +def test_parse_airports_page_keeps_text_fields_as_strings(): + payload = {"props": {"country": {"name": ["Brazil"]}, "airports": [{"name": 42, "iata": None, "icao": {"x": 1}}]}} + airport = parse_airports_page(payload)[0][0] + assert (airport.name, airport.iata, airport.icao, airport.country) == ("", "", "", "") -def test_country_to_slug_takes_a_countries_member(): - """get_airports accepts them, so get_country_flag(airport.country) style calls - must not slugify "Countries.BRAZIL".""" - assert country_to_slug(Countries.BRAZIL) == "brazil" - assert country_to_slug(Countries.MYANMAR_BURMA) == "myanmar-burma" +def test_parse_airports_page_ignores_malformed_states(): + payload = {"props": {"states": [{"url": "/data/airports/x/a"}, {"url": ""}, {"code": "B"}, "junk", {"url": 3}]}} + assert parse_airports_page(payload) == ([], ["/data/airports/x/a"]) -def test_country_to_slug_strips_parentheses_for_flag_urls(): - """These 404'd while the slug was a plain space-to-hyphen replacement.""" - assert country_to_slug("Myanmar (Burma)") == "myanmar-burma" - assert country_to_slug("Cocos (Keeling) Islands") == "cocos-keeling-islands" - assert country_to_slug("Falkland Islands (Malvinas)") == "falkland-islands-malvinas" - assert country_to_slug("Timor-Leste (East Timor)") == "timor-leste-east-timor" +def test_parse_airports_page_invalid_payload_returns_nothing(): + assert parse_airports_page(b"") == ([], []) + assert parse_airports_page(b"not a data page") == ([], []) + assert parse_airports_page({"props": {}}) == ([], []) diff --git a/python/tests/util.py b/python/tests/util.py index 3c03b51..8b887d7 100644 --- a/python/tests/util.py +++ b/python/tests/util.py @@ -12,13 +12,28 @@ def raise_multiple(errors: List) -> None: finally: raise_multiple(errors) -def repeat_test(attempts: int, after: int, errors: Optional[Sequence[Type[BaseException]]] = None) -> Callable: +def is_rate_limited(error: BaseException) -> bool: + """ + Whether FR24 answered 429: shared CI runners trip its rate limit, and a + pause is all that fixes it. + """ + response = getattr(error, "response", None) + return getattr(response, "status_code", None) == 429 or "429" in str(error).split(":")[0] + + +def repeat_test( + attempts: int, + after: int, + errors: Optional[Sequence[Type[BaseException]]] = None, + retry_on: Optional[Callable[[BaseException], bool]] = is_rate_limited, +) -> Callable: """ Decorator to repeat a test N times for specific errors. :param attempts: Number of attempts for testing :param after: Time in seconds to wait for each attempt :param errors: Sequence of exception classes to retry on; if None, retry on any error + :param retry_on: Predicate that also marks an error as retryable, whatever its class """ if errors is not None: invalid = [e for e in errors if not (isinstance(e, type) and issubclass(e, BaseException))] @@ -34,7 +49,9 @@ def wrapper(*args, **kwargs): return test_function(*args, **kwargs) except Exception as error: - if errors is not None and not isinstance(error, tuple(errors)): raise error + retryable = errors is None or isinstance(error, tuple(errors)) + if not retryable and retry_on is not None: retryable = retry_on(error) + if not retryable: raise error if after is not None: time.sleep(after) error_list.append(error) From ea000f2511ce33bc277123f4bffd4921c3e3656c Mon Sep 17 00:00:00 2001 From: JeanExtreme002 Date: Mon, 9 Nov 2026 03:05:54 -0300 Subject: [PATCH 2/2] fix: let the CI finish the airports crawl and clear the brotli advisory - nodejs: the live airports suite crawls one page per country and state and ran past mocha's 10 second default on Node 18 and 20; it now gets 60 seconds, like the Go live tests. - go: govulncheck flags github.com/andybalholm/brotli v1.2.2 (GO-2026-6888); bumped to v1.2.7, which fixes it. --- go/go.mod | 2 +- go/go.sum | 4 ++-- nodejs/tests/testApi.js | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 79d19fc..746c70a 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,7 +3,7 @@ module github.com/JeanExtreme002/FlightRadarAPI/go go 1.26.0 require ( - github.com/andybalholm/brotli v1.2.2 + github.com/andybalholm/brotli v1.2.7 golang.org/x/net v0.62.0 golang.org/x/text v0.43.0 ) diff --git a/go/go.sum b/go/go.sum index 45956a4..4df38b2 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,5 +1,5 @@ -github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= -github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A= +github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= diff --git a/nodejs/tests/testApi.js b/nodejs/tests/testApi.js index c20ddaa..caa7db9 100644 --- a/nodejs/tests/testApi.js +++ b/nodejs/tests/testApi.js @@ -44,6 +44,10 @@ describe("Testing FlightRadarAPI version " + version, function() { }); describe("Getting Airports", function() { + // The listing is one page per country and state, crawled maxWorkers at + // a time: the whole index takes well over mocha's default 10 seconds. + this.timeout(60000); + const expected = 1800; const countries = [Countries.BRAZIL, Countries.UNITED_STATES];