whatcanGOwrong

go/pkg/mod/golang.org/x/telemetry@v0.0.0-20240829154258-f29ab539cc98/internal/mmap/mmap.go

@@ -0,0 +1,36 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package is a lightly modified version of the mmap code
+// in github.com/google/codesearch/index.
+
+// The mmap package provides an abstraction for memory mapping files
+// on different platforms.
+package mmap
+
+import (
+	"os"
+)
+
+// The backing file is never closed, so Data
+// remains valid for the lifetime of the process.
+type Data struct {
+	// TODO(pjw): might be better to define versions of Data
+	// for the 3 specializations
+	f    *os.File
+	Data []byte
+	// Some windows magic
+	Windows interface{}
+}
+
+// Mmap maps the given file into memory.
+// When remapping a file, pass the most recently returned Data.
+func Mmap(f *os.File) (*Data, error) {
+	return mmapFile(f)
+}
+
+// Munmap unmaps the given file from memory.
+func Munmap(d *Data) error {
+	return munmapFile(d)
+}

go/pkg/mod/golang.org/x/telemetry@v0.0.0-20240829154258-f29ab539cc98/internal/mmap/mmap_other.go

@@ -0,0 +1,25 @@
+// Copyright 2022 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build (js && wasm) || wasip1 || plan9 || (solaris && !go1.20)
+
+package mmap
+
+import (
+	"io"
+	"os"
+)
+
+// mmapFile on other systems doesn't mmap the file. It just reads everything.
+func mmapFile(f *os.File) (*Data, error) {
+	b, err := io.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	return &Data{f, b, nil}, nil
+}
+
+func munmapFile(_ *Data) error {
+	return nil
+}

go/pkg/mod/golang.org/x/telemetry@v0.0.0-20240829154258-f29ab539cc98/internal/mmap/mmap_test.go

@@ -0,0 +1,179 @@
+// Copyright 2024 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mmap_test
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"unsafe"
+
+	"golang.org/x/telemetry/internal/counter"
+	"golang.org/x/telemetry/internal/mmap"
+	"golang.org/x/telemetry/internal/testenv"
+)
+
+// If the sharedFileEnv environment variable is set,
+// increment an atomic value in that file rather than
+// run the test.
+const sharedFileEnv = "MMAP_TEST_SHARED_FILE"
+
+func TestMain(m *testing.M) {
+	if name := os.Getenv(sharedFileEnv); name != "" {
+		_, mapping, err := openMapped(name)
+		if err != nil {
+			log.Fatalf("openMapped failed: %v", err)
+		}
+
+		v := (*atomic.Uint64)(unsafe.Pointer(&mapping.Data[0]))
+		v.Add(1)
+		// Exit without explicitly calling munmap/close.
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}
+
+func openMapped(name string) (*os.File, *mmap.Data, error) {
+	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, nil, fmt.Errorf("open failed: %v", err)
+	}
+	data, err := mmap.Mmap(f)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Mmap failed: %v", err)
+	}
+	return f, data, nil
+}
+
+// Via golang/go#68389 and golang/go#68458, we learned that 64-bit atomics were
+// unreliable on linux/arm in Go 1.21. This was fixed in
+// https://go.dev/cl/525637, but only for ARMv7 and later.
+func skipIfLinuxArm(t *testing.T) {
+	if runtime.GOOS == "linux" && runtime.GOARCH == "arm" {
+		t.Skipf("64-bit atomics may not work on linux/arm")
+	}
+}
+
+func TestSharedMemory(t *testing.T) {
+	testenv.SkipIfUnsupportedPlatform(t)
+	skipIfLinuxArm(t)
+
+	// This test verifies that Mmap'ed files are usable for concurrent
+	// cross-process atomic operations.
+
+	dir := t.TempDir()
+	name := filepath.Join(dir, "shared.count")
+
+	var zero [8]byte
+	if err := os.WriteFile(name, zero[:], 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Fork+exec the current test process.
+	// Child processes atomically increment the counter file in shared memory.
+
+	exe, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const concurrency = 100
+	var wg sync.WaitGroup
+	env := append(os.Environ(), sharedFileEnv+"="+name)
+	for i := 0; i < concurrency; i++ {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cmd := exec.Command(exe)
+			cmd.Env = env
+
+			if err := cmd.Run(); err != nil {
+				t.Errorf("subcommand #%d failed: %v", i, err)
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	data, err := counter.ReadMapped(name)
+	if err != nil {
+		t.Fatalf("final read failed: %v", err)
+	}
+	v := (*atomic.Uint64)(unsafe.Pointer(&data[0]))
+	if got := v.Load(); got != concurrency {
+		t.Errorf("incremented %d times, want %d", got, concurrency)
+	}
+}
+
+func TestMultipleMaps(t *testing.T) {
+	testenv.SkipIfUnsupportedPlatform(t)
+	skipIfLinuxArm(t)
+
+	// This test verifies that multiple views of an mmapp'ed file may
+	// simultaneously exist for the current process. This is relied upon by
+	// counter concurrency logic.
+
+	dir := t.TempDir()
+	name := filepath.Join(dir, "shared.count")
+
+	var zero [8]byte
+	if err := os.WriteFile(name, zero[:], 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	var (
+		mappings []*mmap.Data
+		values   []*atomic.Uint64 // mapped counts
+	)
+
+	const nMaps = 3
+	for i := 0; i < nMaps; i++ {
+		f, mapping, err := openMapped(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		mappings = append(mappings, mapping)
+		i := i
+		defer func() {
+			if i > 0 {
+				mmap.Munmap(mapping)
+			}
+			f.Close()
+		}()
+		values = append(values, (*atomic.Uint64)(unsafe.Pointer(&mapping.Data[0])))
+	}
+
+	var wg sync.WaitGroup
+	const nAdds = 100
+	for _, v := range values {
+		v := v
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				v.Add(1)
+			}
+		}()
+	}
+	wg.Wait()
+	for i, v := range values {
+		if got, want := v.Load(), uint64(nMaps*nAdds); got != want {
+			t.Errorf("counter %d has value %d, want %d", i, got, want)
+		}
+	}
+	mmap.Munmap(mappings[0]) // other mappings should remain valid
+	for i, v := range values[1:] {
+		if got, want := v.Load(), uint64(nMaps*nAdds); got != want {
+			t.Errorf("counter %d has value %d, want %d", i, got, want)
+		}
+	}
+}

go/pkg/mod/golang.org/x/telemetry@v0.0.0-20240829154258-f29ab539cc98/internal/mmap/mmap_unix.go

@@ -0,0 +1,47 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build unix && (!solaris || go1.20)
+
+package mmap
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"syscall"
+)
+
+func mmapFile(f *os.File) (*Data, error) {
+	st, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	size := st.Size()
+	pagesize := int64(os.Getpagesize())
+	if int64(int(size+(pagesize-1))) != size+(pagesize-1) {
+		return nil, fmt.Errorf("%s: too large for mmap", f.Name())
+	}
+	n := int(size)
+	if n == 0 {
+		return &Data{f, nil, nil}, nil
+	}
+	mmapLength := int(((size + pagesize - 1) / pagesize) * pagesize) // round up to page size
+	data, err := syscall.Mmap(int(f.Fd()), 0, mmapLength, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
+	if err != nil {
+		return nil, &fs.PathError{Op: "mmap", Path: f.Name(), Err: err}
+	}
+	return &Data{f, data[:n], nil}, nil
+}
+
+func munmapFile(d *Data) error {
+	if len(d.Data) == 0 {
+		return nil
+	}
+	err := syscall.Munmap(d.Data)
+	if err != nil {
+		return &fs.PathError{Op: "munmap", Path: d.f.Name(), Err: err}
+	}
+	return nil
+}

go/pkg/mod/golang.org/x/telemetry@v0.0.0-20240829154258-f29ab539cc98/internal/mmap/mmap_windows.go

@@ -0,0 +1,52 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mmap
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+func mmapFile(f *os.File) (*Data, error) {
+	st, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	size := st.Size()
+	if size == 0 {
+		return &Data{f, nil, nil}, nil
+	}
+	// set the min and max sizes to zero to map the whole file, as described in
+	// https://learn.microsoft.com/en-us/windows/win32/memory/creating-a-file-mapping-object#file-mapping-size
+	h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, syscall.PAGE_READWRITE, 0, 0, nil)
+	if err != nil {
+		return nil, fmt.Errorf("CreateFileMapping %s: %w", f.Name(), err)
+	}
+	// the mapping extends from zero to the end of the file mapping
+	// https://learn.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-mapviewoffile
+	addr, err := windows.MapViewOfFile(h, syscall.FILE_MAP_READ|syscall.FILE_MAP_WRITE, 0, 0, 0)
+	if err != nil {
+		return nil, fmt.Errorf("MapViewOfFile %s: %w", f.Name(), err)
+	}
+	// Note: previously, we called windows.VirtualQuery here to get the exact
+	// size of the memory mapped region, but VirtualQuery reported sizes smaller
+	// than the actual file size (hypothesis: VirtualQuery only reports pages in
+	// a certain state, and newly written pages may not be counted).
+	return &Data{f, unsafe.Slice((*byte)(unsafe.Pointer(addr)), size), h}, nil
+}
+
+func munmapFile(d *Data) error {
+	err := windows.UnmapViewOfFile(uintptr(unsafe.Pointer(&d.Data[0])))
+	x, ok := d.Windows.(windows.Handle)
+	if ok {
+		windows.CloseHandle(x)
+	}
+	d.f.Close()
+	return err
+}