tysv

tysv allows building typed yielder from full text (String) or file streams (FileStream). This is particularly useful for huge CSV files that cannot fit in the memory buffer.

gleam add tysv

Standard CSV

import gleam/float
import gleam/int
import gleam/string
import gleam/yielder
import tysv.{type Text}
import tysv/text

const data = "Vanilla,23,0.5
Strawberry,24,0.75
Mango,34,1.75"

const rows = [
  #("Vanilla", 23, 0.5),
  #("Strawberry", 24, 0.75),
  #("Mango", 34, 1.75),
]

pub fn standard_csv() {
  // Build typed yielder from CSV
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.from(Text(data))
    |> yielder.to_list

  // Build CSV from typed yielder
  assert data
    == text.build(fn(row) {
      let #(flavor, price, kg) = row
      [flavor, int.to_string(price), float.to_string(kg)]
    })
    |> text.from(rows |> yielder.from_list)
}

Skip Header

// Dependencies and constants from the standard case are omitted for brevity.

pub fn skip_header() {
  let data = "Flavor,Price,Kg\n" <> data

  // Build typed yielder from CSV (with header)
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.skip_header(True)
    |> tysv.from(Text(data))
    |> yielder.to_list

  // Build CSV (with header) from typed yielder
  assert data
    == text.build(fn(row) {
      let #(flavor, price, kg) = row
      [flavor, int.to_string(price), float.to_string(kg)]
    })
    |> text.headers(["Flavor", "Price", "Kg"])
    |> text.from(rows |> yielder.from_list)
}

Custom Separators

// Dependencies and constants from the standard case are omitted for brevity.

pub fn custom_separators() {
  let data =
    string.replace(data, ",", ";")
    |> string.replace("\n", "|")

  // Build typed yielder from CSV (with custom separators)
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.col_sep(";")
    |> tysv.row_sep("|")
    |> tysv.from(Text(data))
    |> yielder.to_list

  // Build CSV (with custom separators) from typed yielder
  assert data
    == text.build(fn(row) {
      let #(flavor, price, kg) = row
      [flavor, int.to_string(price), float.to_string(kg)]
    })
    |> text.col_sep(";")
    |> text.row_sep("|")
    |> text.from(rows |> yielder.from_list)
}

With remaining columns

pub fn standard_csv() {
  let data =
    data |> string.replace("\n", ",wibble,wobble\n") <> ",wibble,wobble"
  let rows = {
    use row <- list.map(rows)
    #(row, ["wibble", "wobble"])
  }

  // Build typed yielder from CSV
  assert rows
    == tysv.build({
      use flavor <- tysv.parsed
      use price <- tysv.parsed
      use kg <- tysv.parsed
      #(flavor, price, kg)
    })
    |> tysv.col(Ok)
    |> tysv.col(int.parse)
    |> tysv.col(float.parse)
    |> tysv.from_with_rest(Text(data))
    |> yielder.to_list
}

Further documentation can be found at https://hexdocs.pm/tysv.

Development

gleam test  # Run the tests

Provenance

All code in this project is written naturally. No AI generation or assistance is used to implement any functionality.

LICENSE

tysv is made available under the terms of the EUPL-1.2 License.