szczanieckiej/traffic/access.go

package traffic

import (
	"apiote.xyz/p/szczanieckiej/config"
	"apiote.xyz/p/szczanieckiej/file"
	"apiote.xyz/p/szczanieckiej/gtfs_rt"
	traffic_errors "apiote.xyz/p/szczanieckiej/traffic/errors"
	"apiote.xyz/p/szczanieckiej/transformers"

	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/language"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"

	"git.sr.ht/~sircmpwn/go-bare"
	"github.com/dhconnelly/rtreego"
	"github.com/sahilm/fuzzy"
	"notabug.org/apiote/gott"
)

type OlcError struct {
	Value string
	Err   error
}

func (e OlcError) Error() string {
	return e.Err.Error()
}

type _Result struct {
	Filename       string
	Offset         uint
	Date           time.Time
	LineName       string
	TimetableHome  string
	Calendar       []Schedule
	DeparturesType DeparturesType
	Vehicles       Vehicles
	Feed           Feed

	Location          *time.Location
	Datetime          time.Time
	MinuteB4Datetime  time.Time
	Midnight          time.Time
	TodaySchedule     map[string]struct{}
	YesterdaySchedule map[string]struct{}
	file              *os.File
	TripsFile         *os.File
	Trips             map[string]Trip

	Departures []DepartureRealtime
	Stop       Stop
	Line       Line
	Trip       Trip
	FeedInfo   FeedInfo
}

var lastUpdatedGtfsRt = map[string]uint64{}

func isTimeout(err error) bool {
	var e net.Error
	return errors.As(err, &e) && e.Timeout()
}

func CleanQuery(query string, feed Feed) (string, error) {
	t := transform.Chain(runes.Remove(runes.Predicate(transformers.IsNonAlphanum)), feed.Transformer())
	queryCleaned, _, err := transform.String(t, query)
	return strings.ToLower(queryCleaned), err
}

func findSchedule(home string, time time.Time, calendar []Schedule) (map[string]struct{},
	error) {
	schedules := map[string]struct{}{}
	weekday := uint8(1 << time.Weekday())
	date := time.Format(DateFormat)
	for _, schedule := range calendar {
		if schedule.StartDate <= date && date <= schedule.EndDate &&
			(schedule.Weekdays&weekday != 0) {
			schedules[schedule.Id] = struct{}{}
		}
	}
	var err error
	if len(schedules) == 0 {
		err = traffic_errors.NoSchedule{Date: date}
	}
	return schedules, err
}

func getRealtimeOffset(tripID string, stopSequence int,
	feed Feed) (gtfs_rt.Update, error) {
	updates, lastUpdated, err := gtfs_rt.GetRt(lastUpdatedGtfsRt[feed.String()],
		feed.RealtimeFeeds(), feed.String())
	if err != nil {
		return gtfs_rt.Update{}, err
	}
	lastUpdatedGtfsRt[feed.String()] = lastUpdated
	update := updates[tripID]
	return update, nil
}

func getRealtimeUpdates(feed Feed) (map[string]gtfs_rt.Update, error) {
	updates, lastUpdated, err := gtfs_rt.GetRt(lastUpdatedGtfsRt[feed.String()],
		feed.RealtimeFeeds(), feed.String())
	if err != nil {
		return map[string]gtfs_rt.Update{}, err
	}
	lastUpdatedGtfsRt[feed.String()] = lastUpdated
	return updates, nil
}

func calculateGtfsTime(gtfsTime uint, delay int32, date time.Time,
	timezone *time.Location) (time.Time, error) {
	noon := time.Date(date.Year(), date.Month(), date.Day(), 12, 0, 0, 0,
		timezone)
	twelve, _ := time.ParseDuration("-12h")
	midnight := noon.Add(twelve)
	departureDuration, err := time.ParseDuration(
		strconv.FormatInt(int64(gtfsTime), 10) + "m")
	if err != nil {
		return midnight, err
	}
	delayDuration, err := time.ParseDuration(strconv.FormatInt(int64(delay),
		10) + "s")
	if err != nil {
		return midnight, err
	}
	t := midnight.Add(departureDuration).Add(delayDuration)
	return t, nil
}

func loadLocation(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)
	var err error = nil
	result.Location, err = GetTimezone(result.Stop, result.Feed)
	return result, err
}

func loadTime(input ...interface{}) interface{} {
	result := input[0].(_Result)

	deadzone, _ := time.ParseDuration("-1m")
	now := time.Now()
	datetime := time.Date(result.Date.Year(), result.Date.Month(),
		result.Date.Day(), now.Hour(), now.Minute(), now.Second(), 0,
		result.Location)
	result.Datetime = datetime
	result.MinuteB4Datetime = datetime.Add(deadzone)
	result.Midnight = time.Date(datetime.Year(), datetime.Month(),
		datetime.Day(), 0, 0, 0, 0, result.Location)
	return result
}

func loadTodaySchedule(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	todaySchedule, err := findSchedule(result.TimetableHome, result.Date,
		result.Calendar)
	result.TodaySchedule = todaySchedule
	return result, err
}

func loadYesterdaySchedule(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	yesterday := result.Date.AddDate(0, 0, -1)
	yesterdaySchedule, err := findSchedule(result.TimetableHome, yesterday,
		result.Calendar)
	result.YesterdaySchedule = yesterdaySchedule
	return result, err
}

func recoverYesterdaySchedule(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)
	err := input[1].(error)

	dayBefore := result.Date.AddDate(0, 0, -1).Format(DateFormat)
	if err, ok := err.(traffic_errors.NoSchedule); ok && err.Date == dayBefore {
		result.YesterdaySchedule = map[string]struct{}{}
		return gott.Tuple{result}, nil
	}
	return gott.Tuple{result}, err
}

func openFile(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	file, err := os.Open(filepath.Join(result.TimetableHome, result.Filename))
	result.file = file
	return result, err
}

func seek(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	_, err := result.file.Seek(int64(result.Offset), 0)
	return result, err
}

func unmarshalStop(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	result.Stop = Stop{}
	err := bare.UnmarshalReader(result.file, &result.Stop)
	result.file.Close()
	return result, err
}

func unmarshalFeedInfo(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	result.FeedInfo = FeedInfo{}
	err := bare.UnmarshalReader(result.file, &result.FeedInfo)
	result.file.Close()
	return result, err
}

func unmarshalLine(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	result.Line = Line{}
	err := bare.UnmarshalReader(result.file, &result.Line)
	result.file.Close()
	return result, err
}

func unmarshalTrip(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	result.Trip = Trip{}
	err := bare.UnmarshalReader(result.file, &result.Trip)
	result.file.Close()
	return result, err
}

func openTripsFile(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)

	tripsFile, err := os.Open(filepath.Join(result.TimetableHome, "trips.bare"))
	result.TripsFile = tripsFile
	return result, err
}

func readTrips(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)
	trips := map[string]Trip{}
	orders := map[string]StopOrder{}

	for _, order := range result.Stop.Order {
		_, err := result.TripsFile.Seek(int64(order.TripOffset), 0)
		if err != nil {
			return result, err
		}
		trip := Trip{}
		err = bare.UnmarshalReader(result.TripsFile, &trip)
		if err != nil {
			return result, err
		}

		_, inToday := result.TodaySchedule[trip.ScheduleID]
		_, inYesterday := result.YesterdaySchedule[trip.ScheduleID]
		if inToday || inYesterday {
			trips[trip.Id] = trip
			orders[trip.Id] = order
		}
	}
	result.Stop.Order = orders
	result.Trips = trips
	return result, nil
}

func getUpdates(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)
	departures := []DepartureRealtime{}
	timedOut := false
	for tripID, order := range result.Stop.Order {
		trip := result.Trips[tripID]

		var date time.Time
		if _, ok := result.TodaySchedule[trip.ScheduleID]; ok {
			date = result.Date
		} else if _, ok := result.YesterdaySchedule[trip.ScheduleID]; ok {
			date = result.Date.AddDate(0, 0, -1)
		} else {
			continue
		}

		departure, err := getDeparture(date, result, order, trip, result.Feed, timedOut)
		if err != nil {
			if isTimeout(err) {
				timedOut = true
				err = nil
			} else {
				return result, err
			}
		}
		departures = append(departures, departure)
	}
	result.Departures = departures
	result.TripsFile.Close()
	return result, nil
}

func getDeparture(date time.Time, result _Result, order StopOrder,
	trip Trip, feed Feed, timedOut bool) (DepartureRealtime, error) {
	found := false
	departureRt := DepartureRealtime{}
	var finalErr error
	for _, departure := range trip.Departures {
		if departure.StopSequence == order.Sequence {

			departureRt.Departure = departure
			departureRt.Headsign = trip.Headsign
			departureRt.LineName = trip.LineName
			departureRt.Order = order
			departureRt.Update = gtfs_rt.Update{}

			departureTime, err := calculateGtfsTime(departure.Time, 0, date,
				result.Location)
			if err != nil {
				return departureRt, err
			}
			departureRt.Update.Time = departureTime

			if departureTime.After(result.Midnight) {
				if result.DeparturesType == DEPARTURES_HYBRID && !timedOut {
					departureRt.Update, err = getRealtimeOffset(trip.Id,
						order.Sequence, feed)
					if err != nil {
						if isTimeout(err) {
							timedOut = true
							finalErr = err
						} else {
							log.Printf("while getting realtime departures: %v\n", err)
						}
					}
				}
				departureTime, err := calculateGtfsTime(departure.Time,
					departureRt.Update.Delay, date, result.Location)
				if err != nil {
					return departureRt, err
				}
				departureRt.Update.Time = departureTime
			}
			found = true
			break
		}
	}
	if !found {
		return departureRt, traffic_errors.NoStopOrder{
			TripID: trip.Id,
			Order:  order.Sequence,
		}
	}
	return departureRt, finalErr
}

func filterDepartures(input ...interface{}) interface{} {
	result := input[0].(_Result)
	departures := []DepartureRealtime{}
	for _, departure := range result.Departures {
		if result.DeparturesType == DEPARTURES_FULL ||
			departure.Update.Time.After(result.MinuteB4Datetime) {
			departures = append(departures, departure)
		}
	}
	result.Departures = departures
	return result
}

func filterDeparturesByLine(input ...interface{}) interface{} {
	result := input[0].(_Result)
	departures := []DepartureRealtime{}
	if result.LineName != "" {
		for _, departure := range result.Departures {
			if departure.LineName == result.LineName {
				departures = append(departures, departure)
			}
		}
		result.Departures = departures
	}
	return result
}

func sortDepartures(input ...interface{}) interface{} {
	result := input[0].(_Result)

	sort.Slice(result.Departures, func(i, j int) bool {
		return result.Departures[i].Update.Time.Before(
			result.Departures[j].Update.Time)
	})

	return result
}

func closeFiles(input ...interface{}) (interface{}, error) {
	result := input[0].(_Result)
	err := input[1].(error)
	if result.file != nil {
		result.file.Close()
	}
	if result.TripsFile != nil {
		result.TripsFile.Close()
	}
	return result, err
}

func unmarshalCodeIndex(timetableHome string) (CodeIndex, error) {
	ix := CodeIndex{}

	ixFile, err := os.Open(filepath.Join(timetableHome, "ix_stop_codes.bare"))
	if err != nil {
		return ix, fmt.Errorf("while opening file: %w\n", err)
	}
	defer ixFile.Close()

	r := bare.NewReader(ixFile)
	num, err := r.ReadUint()
	if err != nil {
		return ix, fmt.Errorf("while reading length: %w\n", err)
	}
	for i := uint64(0); i < num; i++ {
		k, err := r.ReadString()
		if err != nil {
			return ix, fmt.Errorf("while reading key at %d: %w\n", i, err)
		}
		v, err := r.ReadUint()
		if err != nil {
			return ix, fmt.Errorf("while reading value at %d: %w\n", i, err)
		}
		ix[k] = uint(v)
	}

	return ix, nil
}

func unmarshalNameIndex(timetableHome, filename string) (NameIndex, error) {
	ix := NameIndex{}

	ixFile, err := os.Open(filepath.Join(timetableHome, filename))
	if err != nil {
		return ix, fmt.Errorf("while opening file: %w", err)
	}
	defer ixFile.Close()

	for err == nil {
		nameOffset := NameOffset{}
		err = bare.UnmarshalReader(ixFile, &nameOffset)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				return ix, fmt.Errorf("while unmarshaling: %w", err)
			}
		}
		ix = append(ix, nameOffset)
	}
	return ix, nil
}

func unmarshalLineIndex(timetableHome string) (NameIndex, error) {
	return unmarshalNameIndex(timetableHome, "ix_lines.bare")
}

func unmarshalStopNameIndex(timetableHome string) (NameIndex, error) {
	return unmarshalNameIndex(timetableHome, "ix_stop_names.bare")
}

func unmarshalTripIndex(timetableHome string) (NameIndex, error) {
	return unmarshalNameIndex(timetableHome, "ix_trips.bare")
}

func readIndexes(feedHome string, versions []Version) (FeedCodeIndex,
	FeedNameIndex, FeedNameIndex, FeedNameIndex, error) {
	codeIndex := FeedCodeIndex{}
	nameIndex := FeedNameIndex{}
	lineIndex := FeedNameIndex{}
	tripIndex := FeedNameIndex{}
	for _, v := range versions {
		validity := Validity(v.String())
		timetableHome := filepath.Join(feedHome, string(validity))
		cIx, err := unmarshalCodeIndex(timetableHome)
		if err != nil {
			return codeIndex, nameIndex, lineIndex, tripIndex,
				fmt.Errorf("while unmarshalling code index: %w\n", err)
		}
		nIx, err := unmarshalStopNameIndex(timetableHome)
		if err != nil {
			return codeIndex, nameIndex, lineIndex, tripIndex,
				fmt.Errorf("while unmarshalling name index: %w\n", err)
		}
		lIx, err := unmarshalLineIndex(timetableHome)
		if err != nil {
			return codeIndex, nameIndex, lineIndex, tripIndex,
				fmt.Errorf("while unmarshalling line index: %w\n", err)
		}
		tIx, err := unmarshalTripIndex(timetableHome)
		if err != nil {
			return codeIndex, nameIndex, lineIndex, tripIndex,
				fmt.Errorf("while unmarshalling trip index: %w\n", err)
		}
		codeIndex[validity] = cIx
		nameIndex[validity] = nIx
		lineIndex[validity] = lIx
		tripIndex[validity] = tIx
	}
	return codeIndex, nameIndex, lineIndex, tripIndex, nil
}

func unmarshalCalendar(timetableHome string) ([]Schedule, error) {
	calendar := []Schedule{}

	calendarFile, err := os.Open(filepath.Join(timetableHome, "calendar.bare"))
	if err != nil {
		return calendar, fmt.Errorf("while opening file: %w", err)
	}
	defer calendarFile.Close()

	for err == nil {
		schedule := Schedule{}
		err = bare.UnmarshalReader(calendarFile, &schedule)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				return calendar, fmt.Errorf("while unmarshaling: %w", err)
			}
		}
		calendar = append(calendar, schedule)
	}
	return calendar, nil
}

func readCalendar(feedHome string, versions []Version) (FeedCalendar, error) {
	calendars := FeedCalendar{}
	for _, v := range versions {
		validity := Validity(v.String())
		timetableHome := filepath.Join(feedHome, string(validity))
		schedule, err := unmarshalCalendar(timetableHome)
		if err != nil {
			return calendars, fmt.Errorf("while unmarshaling for %s: %w", v, err)
		}
		calendars[validity] = schedule
	}
	return calendars, nil
}

func unmarshalVehicles(timetableHome string) (Vehicles, error) {
	vehicles := Vehicles{}

	vehiclesFile, err := os.Open(filepath.Join(timetableHome, "vehicles.bare"))
	if err != nil {
		return vehicles, fmt.Errorf("while opening file: %w", err)
	}
	defer vehiclesFile.Close()

	for err == nil {
		vehicle := Vehicle{}
		err = bare.UnmarshalReader(vehiclesFile, &vehicle)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				return vehicles, fmt.Errorf("while unmarshaling: %w", err)
			}
		}
		vehicles[vehicle.Id] = vehicle
	}
	return vehicles, nil
}

func readVehicles(feedHome string, versions []Version) (FeedVehicles, error) {
	vehicles := FeedVehicles{}
	for _, v := range versions {
		validity := Validity(v.String())
		timetableHome := filepath.Join(feedHome, string(validity))
		versionVehicles, err := unmarshalVehicles(timetableHome)
		if err != nil {
			return vehicles, fmt.Errorf("while unmarshaling for %s: %w", v, err)
		}
		vehicles[validity] = versionVehicles
	}
	return vehicles, nil
}

func createPositionIndex(feedHome string, versions []Version) (FeedPositionIndex, error) {
	feedPositionIndex := FeedPositionIndex{}
	for _, v := range versions {
		positionIndex := rtreego.NewTree(2, 25, 50)

		validity := Validity(v.String())
		timetableHome := filepath.Join(feedHome, string(validity))

		stopsFile, err := os.Open(filepath.Join(timetableHome, "stops.bare"))
		if err != nil {
			return feedPositionIndex, fmt.Errorf("while opening stops file: %w", err)
		}
		defer stopsFile.Close()

		for err == nil {
			stop := Stop{}
			err = bare.UnmarshalReader(stopsFile, &stop)
			if err != nil {
				if err == io.EOF {
					break
				} else {
					return feedPositionIndex, fmt.Errorf("while unmarshaling: %w", err)
				}
			}
			stop.Name = ""
			stop.NodeName = ""
			stop.ChangeOptions = nil
			stop.Zone = ""
			stop.Order = nil
			positionIndex.Insert(stop)
			feedPositionIndex[validity] = positionIndex
		}
	}
	return feedPositionIndex, nil
}

func unmarshalTripFromFile(tripsFile *os.File) Trip {
	trip := Trip{}
	_ = bare.UnmarshalReader(tripsFile, &trip)
	return trip
}

func EnableFeeds(cfg config.Config, traffic *Traffic) {
	feedsMap := RegisterFeeds()
	feeds := map[string]Feed{}
	for _, enabledFeed := range cfg.EnabledFeeds {
		feeds[enabledFeed] = feedsMap[enabledFeed]
	}
	traffic.Feeds = feeds
}

func Initialise(sigChan chan os.Signal, doneChan chan bool, cfg config.Config,
	traffic *Traffic) {
	bare.MaxMapSize(8192)
	for {
		sig := <-sigChan
		if sig == os.Interrupt {
			break
		}
		allVersions := GlobalVersions{}
		codeIndexes := GlobalCodeIndex{}
		nameIndexes := GlobalNameIndex{}
		lineIndexes := GlobalNameIndex{}
		tripIndexes := GlobalNameIndex{}
		calendars := GlobalCalendar{}
		vehicles := GlobalVehicles{}
		positionIndexes := GlobalPositionIndex{}
		for _, feed := range traffic.Feeds {
			feedHome := filepath.Join(cfg.FeedsPath, feed.String())
			err := file.UnpackTraffic(cfg.FeedsPath, feed.String())
			if err != nil {
				log.Printf("while unpacking TRAFFIC in feed %s: %v\n", feed, err)
				continue
			}
			err = CleanOldVersions(cfg, feed)
			if err != nil {
				log.Printf("while cleaning old TRAFFIC versions in feed %s: %v\n",
					feed, err)
				continue
			}
			feedVersions, err := ListVersions(cfg, feed)
			if err != nil {
				log.Printf("while listing TRAFFIC versions in feed %s: %v\n", feed,
					err)
				continue
			}

			feedName := feed.String()
			allVersions[feedName] = feedVersions
			codeIndexes[feedName], nameIndexes[feedName], lineIndexes[feedName], tripIndexes[feedName],
				err = readIndexes(feedHome, feedVersions)
			if err != nil {
				log.Printf("while reading indexes in feed %s: %v\n", feed, err)
				continue
			}
			calendars[feedName], err = readCalendar(feedHome, feedVersions)
			if err != nil {
				log.Printf("while reading calendars in feed %s: %v\n", feed, err)
				continue
			}
			vehicles[feedName], err = readVehicles(feedHome, feedVersions)
			if err != nil {
				log.Printf("while reading vehicles in feed %s: %v\n", feed, err)
				continue
			}
			positionIndexes[feedName], err = createPositionIndex(feedHome, feedVersions)
		}
		traffic.CodeIndexes = codeIndexes
		traffic.NameIndexes = nameIndexes
		traffic.LineIndexes = lineIndexes
		traffic.TripIndexes = tripIndexes
		traffic.Versions = allVersions
		traffic.Calendars = calendars
		traffic.Vehicles = vehicles
		traffic.PositionIndexes = positionIndexes
		log.Println("Initialised")
	}
	doneChan <- true
}

func GetDepartures(stopCode string, lineName, dataHome, feedName string,
	versionCode Validity, traffic *Traffic, date time.Time,
	departuresType DeparturesType) ([]DepartureRealtime, error) {

	codeIndex := traffic.CodeIndexes[feedName][versionCode]
	calendar := traffic.Calendars[feedName][versionCode]
	vehicles := traffic.Vehicles[feedName][versionCode]

	result := _Result{
		Offset:         codeIndex[stopCode],
		Filename:       "stops.bare",
		Date:           date,
		LineName:       lineName,
		TimetableHome:  filepath.Join(dataHome, feedName, string(versionCode)),
		Calendar:       calendar,
		DeparturesType: departuresType,
		Vehicles:       vehicles,
		Feed:           traffic.Feeds[feedName],
	}

	r, e := gott.NewResult(result).
		Bind(loadLocation).
		Map(loadTime).
		Bind(loadTodaySchedule).
		Bind(loadYesterdaySchedule).
		Recover(recoverYesterdaySchedule).
		Bind(openFile).
		Bind(seek).
		Bind(unmarshalStop).
		Bind(openTripsFile).
		Bind(readTrips).
		Bind(getUpdates).
		Map(filterDepartures).
		Map(filterDeparturesByLine).
		Map(sortDepartures).
		Recover(closeFiles).
		Finish()

	if e != nil {
		return []DepartureRealtime{}, e
	} else {
		return r.(_Result).Departures, nil
	}

}

func GetTripFromStop(tripID string, stopCode string, context Context, traffic *Traffic) ([]TimedStopStub, error) {
	stubs := []TimedStopStub{}

	var (
		order    = -1
		trip     Trip
		err      error
		baseTime uint = 0
		time     uint = 0
	)

	if stopCode != "" {
		startingStop, err := GetStop(stopCode, context, traffic)
		if err != nil {
			return stubs, fmt.Errorf("while getting starting stop: %w", err)
		}
		tripOffset := -1
		order = -1
		o := startingStop.Order[tripID]
		tripOffset = int(o.TripOffset)
		order = o.Sequence
		if tripOffset == -1 {
			return stubs, fmt.Errorf("trip for starting stop not found")
		}
		trip, err = GetTripByOffset(uint(tripOffset), context, traffic)
		if err != nil {
			return stubs, fmt.Errorf("while getting trip: %w", err)
		}
	} else {
		trip, err = GetTrip(tripID, context, traffic)
		if err != nil {
			return stubs, fmt.Errorf("while getting trip: %w", err)
		}
	}

	for _, departure := range trip.Departures {
		if departure.StopSequence >= order {
			stop, err := getStopByOffset(uint(departure.StopOffset), context, traffic)
			if err != nil {
				return stubs, fmt.Errorf("while getting stop: %w", err)
			}
			if baseTime != 0 {
				time = departure.Time - baseTime
			}
			stubs = append(stubs, TimedStopStub{
				StopStub: StopStub{
					Code:     stop.Code,
					Name:     stop.Name,
					NodeName: stop.NodeName,
					Zone:     stop.Zone,
					OnDemand: departure.Pickup == BY_DRIVER || departure.Dropoff == BY_DRIVER,
				},
				Time: time,
			})
		}
	}

	return stubs, nil
}

func getStopByOffset(offset uint, context Context, traffic *Traffic) (Stop, error) { // todo offset should be uint64 everywhere
	result := _Result{
		Filename:      "stops.bare",
		Offset:        offset,
		TimetableHome: filepath.Join(context.DataHome, context.FeedName, string(context.Version)),
	}
	r, e := gott.NewResult(result).
		Bind(openFile).
		Bind(seek).
		Bind(unmarshalStop).
		Finish()
	if e != nil {
		return Stop{}, e
	} else {
		return r.(_Result).Stop, nil
	}
}

func getLineByOffset(offset uint, dataHome string, feedName string,
	versionCode Validity, traffic *Traffic) (Line, error) {
	result := _Result{
		Filename:      "lines.bare",
		Offset:        offset,
		TimetableHome: filepath.Join(dataHome, feedName, string(versionCode)),
	}
	r, e := gott.NewResult(result).
		Bind(openFile).
		Bind(seek).
		Bind(unmarshalLine).
		Finish()
	if e != nil {
		return Line{}, e
	} else {
		return r.(_Result).Line, nil
	}
}

func getFeedInfo(dataHome string, feedName string,
	versionCode Validity, traffic *Traffic) (FeedInfo, error) {
	result := _Result{
		Filename:      "feed_info.bare",
		TimetableHome: filepath.Join(dataHome, feedName, string(versionCode)),
	}
	r, e := gott.NewResult(result).
		Bind(openFile).
		Bind(unmarshalFeedInfo).
		Finish()
	if e != nil {
		return FeedInfo{}, e
	} else {
		return r.(_Result).FeedInfo, nil
	}
}

func GetTripsByOffset(offsets []uint, context Context, t *Traffic, filter func(Trip) bool) ([]Trip, error) {
	trips := []Trip{}
	file, err := os.Open(filepath.Join(context.DataHome, context.FeedName, string(context.Version), "trips.bare"))
	if err != nil {
		return trips, fmt.Errorf("while opening file: %w", err)
	}
	defer file.Close()

	offsetsSet := map[uint]struct{}{}
	for _, offset := range offsets {
		offsetsSet[offset] = struct{}{}
	}

	for offset := range offsetsSet {
		_, err = file.Seek(int64(offset), 0)
		if err != nil {
			return trips, fmt.Errorf("while seeking to %d: %w", offset, err)
		}
		trip := Trip{}
		err = bare.UnmarshalReader(file, &trip)
		if err != nil {
			return trips, fmt.Errorf("while unmarshalling at %d: %w", offset, err)
		}
		if filter(trip) {
			trips = append(trips, trip)
		}
	}
	return trips, nil
}

func GetTripByOffset(offset uint, context Context, t *Traffic) (Trip, error) {
	result := _Result{
		Filename:      "trips.bare",
		Offset:        offset,
		TimetableHome: filepath.Join(context.DataHome, context.FeedName, string(context.Version)),
	}
	r, e := gott.NewResult(result).
		Bind(openFile).
		Bind(seek).
		Bind(unmarshalTrip).
		Finish()
	if e != nil {
		return Trip{}, e
	} else {
		return r.(_Result).Trip, nil
	}
}

func GetStop(stopCode string, context Context, traffic *Traffic) (Stop, error) {
	codeIndex := traffic.CodeIndexes[context.FeedName][context.Version]
	return getStopByOffset(codeIndex[stopCode], context, traffic)
}

func GetStopStub(stopCode string, lineName string, context Context, traffic *Traffic) (StopStub, error) {
	stop, err := GetStop(stopCode, context, traffic)
	if err != nil {
		return StopStub{}, err
	}

	var trip Trip
	var stopOrder = -1
	for _, order := range stop.Order {
		offset := order.TripOffset
		trip, _ = GetTripByOffset(offset, context, traffic)
		if trip.LineName == lineName {
			stopOrder = order.Sequence
			break
		}
	}
	if stopOrder == -1 {
		return StopStub{}, fmt.Errorf("cannot the stop on given line")
	}

	var departure *Departure
	for _, d := range trip.Departures {
		if d.StopSequence == stopOrder { // todo binary search
			departure = &d
			break
		}
	}
	if departure == nil {
		return StopStub{}, fmt.Errorf("cannot find departure at sequence %d", stopOrder)
	}

	stopStub := StopStub{
		Code:     stop.Code,
		Name:     stop.Name,
		NodeName: stop.NodeName,
		Zone:     stop.Zone,
		OnDemand: departure.Pickup == BY_DRIVER || departure.Dropoff == BY_DRIVER,
	}
	return stopStub, nil
}

func GetLine(name string, context Context, traffic *Traffic) (Line, error) {
	index := traffic.LineIndexes[context.FeedName][context.Version]
	for _, o := range index {
		cleanedName, err := CleanQuery(name, traffic.Feeds[context.FeedName])
		if err != nil {
			return Line{}, err
		}
		if o.Name == cleanedName {
			return getLineByOffset(o.Offsets[0], context.DataHome, context.FeedName, context.Version, traffic)
		}
	}
	return Line{}, nil
}

func GetTrip(id string, context Context, traffic *Traffic) (Trip, error) {
	tripIndex := traffic.TripIndexes[context.FeedName][context.Version]
	for _, o := range tripIndex {
		if o.Name == id {
			return GetTripByOffset(o.Offsets[0], context, traffic)
		}
	}
	return Trip{}, fmt.Errorf("trip by id %s not found", id)
}

func QueryLines(query string, dataHome string, feedName string,
	versionCode Validity, traffic *Traffic) ([]Line, error) {
	lines := []Line{}
	index := traffic.LineIndexes[feedName][versionCode]
	cleanQuery, err := CleanQuery(query, traffic.Feeds[feedName])
	if err != nil {
		return lines, fmt.Errorf("while cleaning query: %w", err)
	}
	results := fuzzy.FindFrom(cleanQuery, index)
	for _, result := range results {
		for _, offset := range index[result.Index].Offsets {
			line, err := getLineByOffset(offset, dataHome, feedName,
				versionCode, traffic)
			if err != nil {
				return lines, fmt.Errorf("while getting line for %s: %w", result.Str, err)
			}
			lines = append(lines, line)
		}
	}
	return lines, nil
}

func QueryStops(query string, context Context, traffic *Traffic) ([]Stop, error) {
	stops := []Stop{}
	nameIndex := traffic.NameIndexes[context.FeedName][context.Version]
	results := fuzzy.FindFrom(query, nameIndex)
	for _, result := range results {
		for _, offset := range nameIndex[result.Index].Offsets {
			stop, err := getStopByOffset(offset, context, traffic)
			if err != nil {
				return stops, err
			}
			stops = append(stops, stop)
		}
	}
	return stops, nil
}

func GetStopsNear(location Position, context Context, traffic *Traffic) ([]Stop, error) {
	stops := []Stop{}
	positionIndex := traffic.PositionIndexes[context.FeedName][context.Version]
	codeIndex := traffic.CodeIndexes[context.FeedName][context.Version]
	spatials := positionIndex.NearestNeighbors(12, rtreego.Point{location.Lat, location.Lon})
	for _, spatial := range spatials {
		stop, err := getStopByOffset(codeIndex[spatial.(Stop).Code], context, traffic)
		if err != nil {
			return stops, fmt.Errorf("while getting stop by offset for %s: %w", spatial.(Stop).Code, err)
		}
		stops = append(stops, stop)
	}
	return stops, nil
}

func GetAlerts(stopCode string, preferredLanguages []language.Tag, context Context, traffic *Traffic) ([]Alert, error) {
	feed := traffic.Feeds[context.FeedName]

	alertPositions := map[uint]struct{}{}

	gtfsAlerts, lastUpdated, err := gtfs_rt.GetAlerts(lastUpdatedGtfsRt[feed.String()], feed.RealtimeFeeds(), feed.String())
	if err != nil {
		if isTimeout(err) {
			return []Alert{}, nil
		}
		return []Alert{}, fmt.Errorf("while geting alerts: %w", err)
	}
	lastUpdatedGtfsRt[feed.String()] = lastUpdated

	stop, err := GetStop(stopCode, context, traffic)
	for _, pos := range gtfsAlerts.ByStop[stop.Id] {
		alertPositions[pos] = struct{}{}
	}
	for _, option := range stop.ChangeOptions {
		line, err := GetLine(option.LineName, context, traffic)
		if err != nil {
			return []Alert{}, fmt.Errorf("while getting line %s: %w", option.LineName, err)
		}
		for _, pos := range gtfsAlerts.ByRoute[line.Id] {
			alertPositions[pos] = struct{}{}
		}
		for _, pos := range gtfsAlerts.ByType[line.Kind.Value()] {
			alertPositions[pos] = struct{}{}
		}
	}
	for _, order := range stop.Order {
		trip, err := GetTripByOffset(order.TripOffset, context, traffic)
		if err != nil {
			return []Alert{}, fmt.Errorf("while getting trip at %d: %w", order.TripOffset, err)
		}
		for _, pos := range gtfsAlerts.ByTrip[trip.Id] {
			alertPositions[pos] = struct{}{}
		}
	}

	alerts := make([]Alert, len(alertPositions))
	i := 0
	for pos := range alertPositions {
		gtfsAlert := gtfsAlerts.Alerts[pos]

		includeAlert := false
		for _, timeRange := range gtfsAlert.TimeRanges {
			start := time.Unix(int64(timeRange.GetStart()), 0)
			var end time.Time
			if timeRange.GetEnd() == 0 {
				end = time.Date(9999, 12, 31, 23, 59, 59, 0, time.Local)
			} else {
				end = time.Unix(int64(timeRange.GetEnd()), 0)
			}
			now := time.Now()
			if !now.Before(start) && !now.After(end) {
				includeAlert = true
			}
		}
		if !includeAlert {
			continue
		}

		alerts[i] = Alert{
			Cause:  int32(gtfsAlert.Cause),  // todo(BAF10)
			Effect: int32(gtfsAlert.Effect), // todo(BAF10)
		}
		alertHeaderLangs := []language.Tag{}
		for lang := range gtfsAlert.Headers {
			alertHeaderLangs = append(alertHeaderLangs, lang)
		}
		m := language.NewMatcher(alertHeaderLangs)
		tag, _, _ := m.Match(preferredLanguages...)
		alerts[i].Header = gtfsAlert.Headers[tag]

		alertDescriptionLangs := []language.Tag{}
		for lang := range gtfsAlert.Descriptions {
			alertDescriptionLangs = append(alertDescriptionLangs, lang)
		}
		m = language.NewMatcher(alertDescriptionLangs)
		tag, _, _ = m.Match(preferredLanguages...)
		alerts[i].Description = gtfsAlert.Descriptions[tag]

		alertUrlLangs := []language.Tag{}
		for lang := range gtfsAlert.URLs {
			alertUrlLangs = append(alertUrlLangs, lang)
		}
		m = language.NewMatcher(alertUrlLangs)
		tag, _, _ = m.Match(preferredLanguages...)
		alerts[i].URL = gtfsAlert.URLs[tag]

		i++
	}

	return alerts, nil
}

func GetTimezone(stop Stop, feed Feed) (*time.Location, error) {
	if stop.Timezone != "" {
		return time.LoadLocation(stop.Timezone)
	}
	return feed.GetLocation(), nil
}

func GetLanguage(ctx Context, t *Traffic) (string, error) {
	feedInfo, err := getFeedInfo(ctx.DataHome, ctx.FeedName, ctx.Version, t)
	return feedInfo.Language, err
}

func CleanOldVersions(cfg config.Config, feed Feed) error {
	allVersions, err := ListVersions(cfg, feed)
	if err != nil {
		return err
	}
	now := time.Now().In(feed.GetLocation())
	versionsMap := map[string]Version{}
	for _, version := range allVersions {
		versionsMap[version.String()] = version
	}

	sort.Slice(allVersions, func(i, j int) bool {
		return allVersions[i].ValidFrom.Before(allVersions[j].ValidFrom)
	})
	validVersions := FindValidVersions(allVersions, now)
	validVersionsMap := map[string]bool{}
	for _, version := range validVersions {
		validVersionsMap[version.String()] = true
	}
	err = file.CleanOldVersions(FeedPath(cfg, feed), validVersionsMap)
	return err
}

func convertVehicle(update gtfs_rt.Update, context Context, t *Traffic) (VehicleStatus, error) {
	vehicles := t.Vehicles[context.FeedName][context.Version]
	tripID := update.TripUpdate.GetTrip().GetTripId()
	trip, err := GetTrip(tripID, context, t)
	if err != nil {
		return VehicleStatus{}, fmt.Errorf("while converting vehicle: %w", err)
	}
	return VehicleStatus{
		Id:           update.VehicleID,
		Position:     Position{float64(update.Latitude), float64(update.Longitude)},
		Capabilities: vehicles[update.VehicleID].Capabilities,
		Speed:        update.Speed,
		Headsign:     trip.Headsign,
		LineName:     trip.LineName,
	}, nil
}

func GetVehicle(tripID string, context Context, t *Traffic) (VehicleStatus, error) {
	vehicle := VehicleStatus{}
	update, err := getRealtimeOffset(tripID, 0, t.Feeds[context.FeedName])
	if err != nil {
		return vehicle, fmt.Errorf("while getting realtime update: %w", err)
	}
	if update.TripUpdate == nil {
		return vehicle, fmt.Errorf("empty realtime update")
	}
	return convertVehicle(update, context, t)
}

func GetStopsIn(lb, rt Position, context Context, traffic *Traffic) ([]Stop, error) {
	// todo limit rect size
	// todo does it take into account rect 179 -> -179 latitude?
	stops := []Stop{}
	positionIndex := traffic.PositionIndexes[context.FeedName][context.Version]
	codeIndex := traffic.CodeIndexes[context.FeedName][context.Version]
	rect, err := rtreego.NewRectFromPoints(rtreego.Point{lb.Lat, lb.Lon}, rtreego.Point{rt.Lat, rt.Lon})
	if err != nil {
		return stops, fmt.Errorf("while creating a rect: %w", err)
	}
	spatials := positionIndex.SearchIntersect(rect)
	for _, spatial := range spatials {
		stop, err := getStopByOffset(codeIndex[spatial.(Stop).Code], context, traffic)
		if err != nil {
			return stops, fmt.Errorf("while getting stop by offset for %s: %w", spatial.(Stop).Code, err)
		}
		stops = append(stops, stop)
	}
	return stops, nil
}

func GetVehiclesIn(lb, rt Position, context Context, t *Traffic) ([]VehicleStatus, error) {
	// todo limit rect size
	vehicles := []VehicleStatus{}
	updates, err := getRealtimeUpdates(t.Feeds[context.FeedName])
	if err != nil {
		return vehicles, err
	}
	for _, update := range updates {
		if rt.Lon < float64(update.Longitude) || lb.Lon > float64(update.Longitude) {
			continue
		}
		lat := float64(update.Latitude)
		if lb.Lat < rt.Lat {
			if lb.Lat < lat && lat < rt.Lat {
				vehicle, err := convertVehicle(update, context, t)
				if err != nil {
					return vehicles, fmt.Errorf("while converting vehicle: %w", err)
				}
				vehicles = append(vehicles, vehicle)
			}
		} else {
			if lat > lb.Lat || lat < rt.Lat {
				vehicle, err := convertVehicle(update, context, t)
				if err != nil {
					return vehicles, fmt.Errorf("while converting vehicle: %w", err)
				}
				vehicles = append(vehicles, vehicle)
			}
		}
	}
	return vehicles, nil
}