From 66a784d59e90854c9f301fc4df08a44fef1c048f Mon Sep 17 00:00:00 2001
From: Philippe Laflamme <philippe.laflamme@gmail.com>
Date: Mon, 4 Mar 2024 18:57:19 -0500
Subject: [PATCH 1/3] test: add a test for large record batches

---
 src/appender/arrow.rs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/appender/arrow.rs b/src/appender/arrow.rs
index 8aa96a2a..050cca2b 100644
--- a/src/appender/arrow.rs
+++ b/src/appender/arrow.rs
@@ -79,4 +79,24 @@ mod test {
         assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::<usize>(), 5);
         Ok(())
     }
+
+    #[test]
+    fn test_append_record_batch_large() -> Result<()> {
+        let record_count = usize::pow(2, 16) + 1;
+        let db = Connection::open_in_memory()?;
+        db.execute_batch("CREATE TABLE foo(id INT)")?;
+        {
+            let id_array = Int32Array::from(vec![42; record_count]);
+            let schema = Schema::new(vec![Field::new("id", DataType::Int32, true)]);
+            let record_batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(id_array)]).unwrap();
+            let mut app = db.appender("foo")?;
+            app.append_record_batch(record_batch)?;
+        }
+        let count = db.query_row("SELECT COUNT(*) FROM foo", [], |row| {
+            let count: usize = row.get(0)?;
+            Ok(count)
+        })?;
+        assert_eq!(count, record_count);
+        Ok(())
+    }
 }

From da4dfe090d53daecd833eb00b805dc73ae887e5c Mon Sep 17 00:00:00 2001
From: Philippe Laflamme <philippe.laflamme@gmail.com>
Date: Mon, 4 Mar 2024 22:46:29 -0500
Subject: [PATCH 2/3] fix: increase data chunk length as necessary

---
 src/appender/arrow.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/appender/arrow.rs b/src/appender/arrow.rs
index 050cca2b..c719ead2 100644
--- a/src/appender/arrow.rs
+++ b/src/appender/arrow.rs
@@ -36,6 +36,9 @@ impl Appender<'_> {
         }
 
         let mut data_chunk = DataChunk::new(&logical_type);
+        if record_batch.num_rows() > data_chunk.len() {
+            data_chunk.set_len(record_batch.num_rows());
+        }
         record_batch_to_duckdb_data_chunk(&record_batch, &mut data_chunk).map_err(|_op| Error::AppendError)?;
 
         let rc = unsafe { duckdb_append_data_chunk(self.app, data_chunk.get_ptr()) };

From 25c43ab0129c8c47bf6e61f916f7849958d3c59a Mon Sep 17 00:00:00 2001
From: Philippe Laflamme <philippe.laflamme@gmail.com>
Date: Thu, 21 Mar 2024 11:04:10 -0400
Subject: [PATCH 3/3] fix: missing use

---
 src/appender/arrow.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/appender/arrow.rs b/src/appender/arrow.rs
index c719ead2..dbb04a05 100644
--- a/src/appender/arrow.rs
+++ b/src/appender/arrow.rs
@@ -50,7 +50,7 @@ impl Appender<'_> {
 mod test {
     use crate::{Connection, Result};
     use arrow::{
-        array::{Int8Array, StringArray},
+        array::{Int32Array, Int8Array, StringArray},
         datatypes::{DataType, Field, Schema},
         record_batch::RecordBatch,
     };