# presumably release it when we OS the model
tokenizer_fn = lambda x: tokenizer(x).input_ids if len(x) > 0 else []

pipeline = IngestionPipeline(
    transformations=[
        SentenceSplitter(chunk_size=512, chunk_overlap=0, tokenizer=tokenizer_fn)
    ]
)