From 0615a2917c932901fe8b1f2353edd51e3b82bc95 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 18 Feb 2023 20:25:25 -0600 Subject: [PATCH] refactoring --- .../fullData/sources/FullDataSource.java | 2 +- .../file/fullDatafile/FullDataMetaFile.java | 48 ++++++----- .../AbstractMetaDataContainerFile.java | 81 ++++++++++--------- 3 files changed, 71 insertions(+), 60 deletions(-) diff --git a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java index 298ac481c..1eeeb77bb 100644 --- a/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java +++ b/core/src/main/java/com/seibel/lod/core/dataObjects/fullData/sources/FullDataSource.java @@ -119,7 +119,7 @@ public class FullDataSource extends FullArrayView implements IFullDataSource else { LodUtil.assertNotReach(); - //TODO; + //TODO } } diff --git a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java index b91906ed0..9a4ce1055 100644 --- a/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/fullDatafile/FullDataMetaFile.java @@ -139,26 +139,30 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile // return (getData == null ? 0 : this.metaData.dataVersion.get()) - cacheVersion <= 0; // } // } - - public void addToWriteQueue(ChunkSizedFullDataSource datatype) { + + public void addToWriteQueue(ChunkSizedFullDataSource datatype) + { debugCheck(); DhLodPos chunkPos = new DhLodPos((byte) (datatype.dataDetail + 4), datatype.x, datatype.z); - LodUtil.assertTrue(pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos {} doesn't overlap with section {}", chunkPos, pos); + LodUtil.assertTrue(pos.getSectionBBoxPos().overlaps(chunkPos), "Chunk pos "+chunkPos+" doesn't overlap with section "+pos); //LOGGER.info("Write Chunk {} to file {}", chunkPos, pos); - - GuardedMultiAppendQueue queue = writeQueue.get(); + + GuardedMultiAppendQueue writeQueue = this.writeQueue.get(); // Using read lock is OK, because the queue's underlying data structure is thread-safe. // This lock is only used to insure on polling the queue, that the queue is not being // modified by another thread. - Lock appendLock = queue.appendLock.readLock(); + Lock appendLock = writeQueue.appendLock.readLock(); appendLock.lock(); - try { - queue.queue.add(datatype); - } finally { + try + { + writeQueue.queue.add(datatype); + } + finally + { appendLock.unlock(); } } - + // Cause: Generic Type runtime casting cannot safety check it. // However, the Union type ensures the 'data' should only contain the listed type. public CompletableFuture loadOrGetCachedAsync() @@ -333,9 +337,9 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile _backQueue = queue; } - private void saveChanges(IFullDataSource data) + private void saveChanges(IFullDataSource fullDataSource) { - if (data.isEmpty()) + if (fullDataSource.isEmpty()) { if (file.exists() && !file.delete()) { @@ -350,24 +354,24 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile { // Write/Update data LodUtil.assertTrue(metaData != null); - metaData.dataLevel = data.getDataDetail(); - loader = AbstractFullDataSourceLoader.getLoader(data.getClass(), data.getDataVersion()); - LodUtil.assertTrue(loader != null, "No loader for {} (v{})", data.getClass(), data.getDataVersion()); - dataType = data.getClass(); - metaData.dataTypeId = loader == null ? 0 : loader.datatypeId; - metaData.loaderVersion = data.getDataVersion(); - super.writeData((out) -> data.saveData(level, this, out)); + metaData.dataLevel = fullDataSource.getDataDetail(); + loader = AbstractFullDataSourceLoader.getLoader(fullDataSource.getClass(), fullDataSource.getDataVersion()); + LodUtil.assertTrue(loader != null, "No loader for "+fullDataSource.getClass()+" (v"+fullDataSource.getDataVersion()+")"); + dataType = fullDataSource.getClass(); + metaData.dataTypeId = (loader == null) ? 0 : loader.datatypeId; + metaData.loaderVersion = fullDataSource.getDataVersion(); + super.writeData((outputStream) -> fullDataSource.saveData(level, this, outputStream)); doesFileExist = true; } catch (IOException e) { - LOGGER.error("Failed to save updated data file at {} for sect {}", file, pos, e); + LOGGER.error("Failed to save updated data file at "+file+" for sect "+pos, e); } } } // Return whether any write has happened to the data - private boolean applyWriteQueue(IFullDataSource data) + private boolean applyWriteQueue(IFullDataSource fullDataSource) { // Poll the write queue // First check if write queue is empty, then swap the write queue. @@ -379,7 +383,7 @@ public class FullDataMetaFile extends AbstractMetaDataContainerFile int count = this._backQueue.queue.size(); for (ChunkSizedFullDataSource chunk : this._backQueue.queue) { - data.update(chunk); + fullDataSource.update(chunk); } this._backQueue.queue.clear(); //LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count); diff --git a/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java b/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java index a47669b77..295da9e46 100644 --- a/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java +++ b/core/src/main/java/com/seibel/lod/core/file/metaData/AbstractMetaDataContainerFile.java @@ -102,7 +102,7 @@ public abstract class AbstractMetaDataContainerFile this.file = file; if (!file.exists()) { - throw new FileNotFoundException("File not found at ["+ file +"]"); + throw new FileNotFoundException("File not found at ["+file+"]"); } validateMetaDataFile(this.file); @@ -125,7 +125,15 @@ public abstract class AbstractMetaDataContainerFile int idBytes = byteBuffer.getInt(); if (idBytes != METADATA_IDENTITY_BYTES) { - throw new IOException("Invalid file format: Metadata Identity byte check failed. Expected: ["+METADATA_IDENTITY_BYTES+"], Actual: ["+idBytes+"]."); + if (file.exists()) + { + FileUtil.renameCorruptedFile(file); + throw new IOException("Invalid file format: Metadata Identity byte check failed. Expected: ["+METADATA_IDENTITY_BYTES+"], Actual: ["+idBytes+"]."); + } + else + { + throw new IOException("No file found for meta data. Expected file path: "+file.getPath()); + } } int x = byteBuffer.getInt(); @@ -172,7 +180,7 @@ public abstract class AbstractMetaDataContainerFile } } - protected void writeData(IMetaDataWriter dataWriter) throws IOException + protected void writeData(IMetaDataWriterFunc dataWriterFunc) throws IOException { LodUtil.assertTrue(this.metaData != null); if (this.file.exists()) @@ -184,52 +192,51 @@ public abstract class AbstractMetaDataContainerFile if (USE_ATOMIC_MOVE_REPLACE) { tempFile = new File(this.file.getPath() + ".tmp"); - tempFile.deleteOnExit(); + //tempFile.deleteOnExit(); } else { tempFile = this.file; } - try (FileChannel file = FileChannel.open(tempFile.toPath(), - StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) + try (FileChannel file = FileChannel.open(tempFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { - { - file.position(METADATA_SIZE); - int checksum; - try (OutputStream channelOut = new UnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel - BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer? - CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) - { - // TODO: Is Adler32 ok? - dataWriter.writeBufferToFile(checkedOut); - checksum = (int) checkedOut.getChecksum().getValue(); - } - - file.position(0); - // Write metadata - ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE); - buff.putInt(METADATA_IDENTITY_BYTES); - buff.putInt(this.pos.sectionX); - buff.putInt(Integer.MIN_VALUE); // Unused - buff.putInt(this.pos.sectionZ); - buff.putInt(checksum); - buff.put(this.pos.sectionDetailLevel); - buff.put(this.metaData.dataLevel); - buff.put(this.metaData.loaderVersion); - buff.put(Byte.MIN_VALUE); // Unused - buff.putLong(this.metaData.dataTypeId); - buff.putLong(Long.MAX_VALUE); //buff.putLong(this.metaData.dataVersion.get()); // not currently implemented - LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE); - buff.flip(); - file.write(buff); + file.position(METADATA_SIZE); + int checksum; + try (OutputStream channelOut = new UnclosableOutputStream(Channels.newOutputStream(file)); // Prevent closing the channel + BufferedOutputStream bufferedOut = new BufferedOutputStream(channelOut); // TODO: Is default buffer size ok? Do we even need to buffer? + CheckedOutputStream checkedOut = new CheckedOutputStream(bufferedOut, new Adler32())) + { + // TODO: Is Adler32 ok? + dataWriterFunc.writeBufferToFile(checkedOut); + checksum = (int) checkedOut.getChecksum().getValue(); } + file.position(0); + // Write metadata + ByteBuffer buff = ByteBuffer.allocate(METADATA_SIZE); + buff.putInt(METADATA_IDENTITY_BYTES); + buff.putInt(this.pos.sectionX); + buff.putInt(Integer.MIN_VALUE); // Unused + buff.putInt(this.pos.sectionZ); + buff.putInt(checksum); + buff.put(this.pos.sectionDetailLevel); + buff.put(this.metaData.dataLevel); + buff.put(this.metaData.loaderVersion); + buff.put(Byte.MIN_VALUE); // Unused + buff.putLong(this.metaData.dataTypeId); + buff.putLong(Long.MAX_VALUE); //buff.putLong(this.metaData.dataVersion.get()); // not currently implemented + LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE); + buff.flip(); + file.write(buff); + + file.close(); if (USE_ATOMIC_MOVE_REPLACE) { // Atomic move / replace the actual file - Files.move(tempFile.toPath(), this.file.toPath(), StandardCopyOption.ATOMIC_MOVE); // TODO couldn't StandardCopyOption.REPLACE_EXISTING also work here? + Files.move(tempFile.toPath(), this.file.toPath(), StandardCopyOption.REPLACE_EXISTING); // TODO couldn't StandardCopyOption. also work here? + LOGGER.info("replaced file: "+this.file.toPath()); } } finally @@ -266,7 +273,7 @@ public abstract class AbstractMetaDataContainerFile //================// @FunctionalInterface - public interface IMetaDataWriter + public interface IMetaDataWriterFunc { void writeBufferToFile(T t) throws IOException; }