[llvm] r269682 - ThinLTO caching: reload cached file with mmap and drop heap-allocated memory buffer

[Duncan P. N. Exon Smith via llvm-commits]

> On 2016-May-16, at 12:12, Mehdi Amini via llvm-commits <llvm-commits at lists.llvm.org> wrote:
> 
> Author: mehdi_amini
> Date: Mon May 16 14:11:59 2016
> New Revision: 269682
> 
> URL: http://llvm.org/viewvc/llvm-project?rev=269682&view=rev
> Log:
> ThinLTO caching: reload cached file with mmap and drop heap-allocated memory buffer
> 
> This is reducing pressure on the OS memory system, and is NFC
> when not using a cache.
> I measure a 10x memory consumption reduction when linking opt
> with full debug info.
> 
> From: Mehdi Amini <mehdi.amini at apple.com>
> 
> Modified:
>    llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp
> 
> Modified: llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp
> URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp?rev=269682&r1=269681&r2=269682&view=diff
> ==============================================================================
> --- llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp (original)
> +++ llvm/trunk/lib/LTO/ThinLTOCodeGenerator.cpp Mon May 16 14:11:59 2016
> @@ -484,9 +484,10 @@ public:
>   }
> 
>   // Cache the Produced object file
> -  void write(MemoryBufferRef OutputBuffer) {
> +  std::unique_ptr<MemoryBuffer>
> +  write(std::unique_ptr<MemoryBuffer> OutputBuffer) {
>     if (EntryPath.empty())
> -      return;
> +      return OutputBuffer;
> 
>     // Write to a temporary to avoid race condition
>     SmallString<128> TempFilename;
> @@ -499,7 +500,7 @@ public:
>     }
>     {
>       raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
> -      OS << OutputBuffer.getBuffer();
> +      OS << OutputBuffer->getBuffer();
>     }
>     // Rename to final destination (hopefully race condition won't matter here)
>     EC = sys::fs::rename(TempFilename, EntryPath);
> @@ -509,8 +510,16 @@ public:
>       if (EC)
>         report_fatal_error(Twine("Failed to open ") + EntryPath +
>                            " to save cached entry\n");
> -      OS << OutputBuffer.getBuffer();
> +      OS << OutputBuffer->getBuffer();
>     }
> +    auto ReloadedBufferOrErr = MemoryBuffer::getFile(EntryPath);
> +    if (auto EC = ReloadedBufferOrErr.getError()) {
> +      // FIXME diagnose

Do you have a plan to resolve this FIXME?

> +      errs() << "error: can't reload cached file '" << EntryPath
> +             << "': " << EC.message() << "\n";
> +      return OutputBuffer;
> +    }
> +    return std::move(*ReloadedBufferOrErr);
>   }
> };
> 
> @@ -943,7 +952,7 @@ void ThinLTOCodeGenerator::run() {
>             ExportList, GUIDPreservedSymbols, ResolvedODR, CacheOptions,
>             DisableCodeGen, SaveTempsDir, count);
> 
> -        CacheEntry.write(*OutputBuffer);
> +        OutputBuffer = CacheEntry.write(std::move(OutputBuffer));
>         ProducedBinaries[count] = std::move(OutputBuffer);
>       }, count);
>       count++;
> 
> 
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits