Deps update, gopkg.lock fix, flagger initialization fix.

This commit is contained in:
2019-02-25 16:29:48 +05:00
parent 23510e22a0
commit afb076d116
406 changed files with 55802 additions and 18666 deletions

View File

@@ -1,26 +1,25 @@
FROM ubuntu:16.04
# Use the most recent ubuntu sources
RUN echo 'deb http://en.archive.ubuntu.com/ubuntu/ artful main universe' >> /etc/apt/sources.list
FROM ubuntu:18.10
# Dependencies to get the git sources and go binaries
RUN apt-get update && apt-get install -y \
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Get the git sources. If not cached, this takes O(5 minutes).
WORKDIR /git
RUN git config --global advice.detachedHead false
# Linux Kernel: Released 03 Sep 2017
RUN git clone --branch v4.13 --depth 1 https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux
# GNU C library: Released 02 Aug 2017 (we should try to get a secure way to clone this)
RUN git clone --branch glibc-2.26 --depth 1 git://sourceware.org/git/glibc.git
# Linux Kernel: Released 23 Dec 2018
RUN git clone --branch v4.20 --depth 1 https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux
# GNU C library: Released 01 Feb 2019 (we should try to get a secure way to clone this)
RUN git clone --branch release/2.29/master --depth 1 git://sourceware.org/git/glibc.git
# Get Go 1.9.2
ENV GOLANG_VERSION 1.9.2
# Get Go
ENV GOLANG_VERSION 1.12beta2
ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
ENV GOLANG_DOWNLOAD_SHA256 de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b
ENV GOLANG_DOWNLOAD_SHA256 9e4884b46a72e0558187a8af6e8733e039432df1b755f14b361f18b63fa5a63e
RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
&& echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \
@@ -29,20 +28,23 @@ RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
ENV PATH /usr/local/go/bin:$PATH
# Linux and Glibc build dependencies
RUN apt-get update && apt-get install -y \
gawk make python \
# Linux and Glibc build dependencies and emulator
RUN apt-get update && apt-get install -y --no-install-recommends \
bison gawk make python3 \
gcc gcc-multilib \
gettext texinfo \
qemu-user \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Emulator and cross compilers
# Cross compilers (install recommended packages to get cross libc-dev)
RUN apt-get update && apt-get install -y \
qemu \
gcc-aarch64-linux-gnu gcc-arm-linux-gnueabi \
gcc-mips-linux-gnu gcc-mips64-linux-gnuabi64 \
gcc-mips64el-linux-gnuabi64 gcc-mipsel-linux-gnu \
gcc-powerpc64-linux-gnu gcc-powerpc64le-linux-gnu \
gcc-riscv64-linux-gnu \
gcc-s390x-linux-gnu gcc-sparc64-linux-gnu \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Let the scripts know they are in the docker environment

View File

@@ -3,7 +3,7 @@
// license that can be found in the LICENSE file.
// linux/mkall.go - Generates all Linux zsysnum, zsyscall, zerror, and ztype
// files for all 11 linux architectures supported by the go compiler. See
// files for all Linux architectures supported by the go compiler. See
// README.md for more information about the build system.
// To run it you must have a git checkout of the Linux kernel and glibc. Once
@@ -17,6 +17,9 @@ package main
import (
"bufio"
"bytes"
"debug/elf"
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
@@ -49,8 +52,9 @@ type target struct {
Bits int
}
// List of the 11 Linux targets supported by the go compiler. sparc64 is not
// currently supported, though a port is in progress.
// List of all Linux targets supported by the go compiler. Currently, riscv64
// and sparc64 are not fully supported, but there is enough support already to
// generate Go type and error definitions.
var targets = []target{
{
GoArch: "386",
@@ -116,6 +120,12 @@ var targets = []target{
GNUArch: "powerpc64le-linux-gnu",
Bits: 64,
},
{
GoArch: "riscv64",
LinuxArch: "riscv",
GNUArch: "riscv64-linux-gnu",
Bits: 64,
},
{
GoArch: "s390x",
LinuxArch: "s390",
@@ -124,13 +134,13 @@ var targets = []target{
SignedChar: true,
Bits: 64,
},
// {
// GoArch: "sparc64",
// LinuxArch: "sparc",
// GNUArch: "sparc64-linux-gnu",
// BigEndian: true,
// Bits: 64,
// },
{
GoArch: "sparc64",
LinuxArch: "sparc",
GNUArch: "sparc64-linux-gnu",
BigEndian: true,
Bits: 64,
},
}
// ptracePairs is a list of pairs of targets that can, in some cases,
@@ -193,22 +203,36 @@ func makeCommand(name string, args ...string) *exec.Cmd {
return cmd
}
// Set GOARCH for target and build environments.
func (t *target) setTargetBuildArch(cmd *exec.Cmd) {
// Set GOARCH_TARGET so command knows what GOARCH is..
cmd.Env = append(os.Environ(), "GOARCH_TARGET="+t.GoArch)
// Set GOARCH to host arch for command, so it can run natively.
for i, s := range cmd.Env {
if strings.HasPrefix(s, "GOARCH=") {
cmd.Env[i] = "GOARCH=" + BuildArch
}
}
}
// Runs the command, pipes output to a formatter, pipes that to an output file.
func (t *target) commandFormatOutput(formatter string, outputFile string,
name string, args ...string) (err error) {
mainCmd := makeCommand(name, args...)
if name == "mksyscall" {
args = append([]string{"run", "mksyscall.go"}, args...)
mainCmd = makeCommand("go", args...)
t.setTargetBuildArch(mainCmd)
} else if name == "mksysnum" {
args = append([]string{"run", "linux/mksysnum.go"}, args...)
mainCmd = makeCommand("go", args...)
t.setTargetBuildArch(mainCmd)
}
fmtCmd := makeCommand(formatter)
if formatter == "mkpost" {
fmtCmd = makeCommand("go", "run", "mkpost.go")
// Set GOARCH_TARGET so mkpost knows what GOARCH is..
fmtCmd.Env = append(os.Environ(), "GOARCH_TARGET="+t.GoArch)
// Set GOARCH to host arch for mkpost, so it can run natively.
for i, s := range fmtCmd.Env {
if strings.HasPrefix(s, "GOARCH=") {
fmtCmd.Env[i] = "GOARCH=" + BuildArch
}
}
t.setTargetBuildArch(fmtCmd)
}
// mainCmd | fmtCmd > outputFile
@@ -253,6 +277,10 @@ func (t *target) generateFiles() error {
if t.LinuxArch == "powerpc" {
qemuArchName = t.GoArch
}
// Fake uname for QEMU to allow running on Host kernel version < 4.15
if t.LinuxArch == "riscv" {
os.Setenv("QEMU_UNAME", "4.15")
}
os.Setenv("GORUN", "qemu-"+qemuArchName)
} else {
os.Setenv("CC", "gcc")
@@ -292,7 +320,7 @@ func (t *target) generateFiles() error {
return nil
}
// Create the Linux and glibc headers in the include directory.
// Create the Linux, glibc and ABI (C compiler convention) headers in the include directory.
func (t *target) makeHeaders() error {
// Make the Linux headers we need for this architecture
linuxMake := makeCommand("make", "headers_install", "ARCH="+t.LinuxArch, "INSTALL_HDR_PATH="+TempDir)
@@ -327,6 +355,114 @@ func (t *target) makeHeaders() error {
file.Close()
}
// ABI headers will specify C compiler behavior for the target platform.
return t.makeABIHeaders()
}
// makeABIHeaders generates C header files based on the platform's calling convention.
// While many platforms have formal Application Binary Interfaces, in practice, whatever the
// dominant C compilers generate is the de-facto calling convention.
//
// We generate C headers instead of a Go file, so as to enable references to the ABI from Cgo.
func (t *target) makeABIHeaders() (err error) {
abiDir := filepath.Join(IncludeDir, "abi")
if err = os.Mkdir(abiDir, os.ModePerm); err != nil {
return err
}
cc := os.Getenv("CC")
if cc == "" {
return errors.New("CC (compiler) env var not set")
}
// Build a sacrificial ELF file, to mine for C compiler behavior.
binPath := filepath.Join(TempDir, "tmp_abi.o")
bin, err := t.buildELF(cc, cCode, binPath)
if err != nil {
return fmt.Errorf("cannot build ELF to analyze: %v", err)
}
defer bin.Close()
defer os.Remove(binPath)
// Right now, we put everything in abi.h, but we may change this later.
abiFile, err := os.Create(filepath.Join(abiDir, "abi.h"))
if err != nil {
return err
}
defer func() {
if cerr := abiFile.Close(); cerr != nil && err == nil {
err = cerr
}
}()
if err = t.writeBitFieldMasks(bin, abiFile); err != nil {
return fmt.Errorf("cannot write bitfield masks: %v", err)
}
return nil
}
func (t *target) buildELF(cc, src, path string) (*elf.File, error) {
// Compile the cCode source using the set compiler - we will need its .data section.
// Do not link the binary, so that we can find .data section offsets from the symbol values.
ccCmd := makeCommand(cc, "-o", path, "-gdwarf", "-x", "c", "-c", "-")
ccCmd.Stdin = strings.NewReader(src)
ccCmd.Stdout = os.Stdout
if err := ccCmd.Run(); err != nil {
return nil, fmt.Errorf("compiler error: %v", err)
}
bin, err := elf.Open(path)
if err != nil {
return nil, fmt.Errorf("cannot read ELF file %s: %v", path, err)
}
return bin, nil
}
func (t *target) writeBitFieldMasks(bin *elf.File, out io.Writer) error {
symbols, err := bin.Symbols()
if err != nil {
return fmt.Errorf("getting ELF symbols: %v", err)
}
var masksSym *elf.Symbol
for _, sym := range symbols {
if sym.Name == "masks" {
masksSym = &sym
}
}
if masksSym == nil {
return errors.New("could not find the 'masks' symbol in ELF symtab")
}
dataSection := bin.Section(".data")
if dataSection == nil {
return errors.New("ELF file has no .data section")
}
data, err := dataSection.Data()
if err != nil {
return fmt.Errorf("could not read .data section: %v\n", err)
}
var bo binary.ByteOrder
if t.BigEndian {
bo = binary.BigEndian
} else {
bo = binary.LittleEndian
}
// 64 bit masks of type uint64 are stored in the data section starting at masks.Value.
// Here we are running on AMD64, but these values may be big endian or little endian,
// depending on target architecture.
for i := uint64(0); i < 64; i++ {
off := masksSym.Value + i*8
// Define each mask in native by order, so as to match target endian.
fmt.Fprintf(out, "#define BITFIELD_MASK_%d %dULL\n", i, bo.Uint64(data[off:off+8]))
}
return nil
}
@@ -336,7 +472,7 @@ func (t *target) makeZSysnumFile() error {
unistdFile := filepath.Join(IncludeDir, "asm/unistd.h")
args := append(t.cFlags(), unistdFile)
return t.commandFormatOutput("gofmt", zsysnumFile, "linux/mksysnum.pl", args...)
return t.commandFormatOutput("gofmt", zsysnumFile, "mksysnum", args...)
}
// makes the zsyscall_linux_$GOARCH.go file
@@ -351,7 +487,7 @@ func (t *target) makeZSyscallFile() error {
args := append(t.mksyscallFlags(), "-tags", "linux,"+t.GoArch,
"syscall_linux.go", archSyscallFile)
return t.commandFormatOutput("gofmt", zsyscallFile, "./mksyscall.pl", args...)
return t.commandFormatOutput("gofmt", zsyscallFile, "mksyscall", args...)
}
// makes the zerrors_linux_$GOARCH.go file
@@ -397,7 +533,7 @@ func (t *target) mksyscallFlags() (flags []string) {
}
}
// This flag menas a 64-bit value should use (even, odd)-pair.
// This flag means a 64-bit value should use (even, odd)-pair.
if t.GoArch == "arm" || (t.LinuxArch == "mips" && t.Bits == 32) {
flags = append(flags, "-arm")
}
@@ -480,3 +616,160 @@ func writeOnePtrace(w io.Writer, arch, def string) {
fmt.Fprintf(w, "\treturn ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs)))\n")
fmt.Fprintf(w, "}\n")
}
// cCode is compiled for the target architecture, and the resulting data section is carved for
// the statically initialized bit masks.
const cCode = `
// Bit fields are used in some system calls and other ABIs, but their memory layout is
// implementation-defined [1]. Even with formal ABIs, bit fields are a source of subtle bugs [2].
// Here we generate the offsets for all 64 bits in an uint64.
// 1: http://en.cppreference.com/w/c/language/bit_field
// 2: https://lwn.net/Articles/478657/
#include <stdint.h>
struct bitfield {
union {
uint64_t val;
struct {
uint64_t u64_bit_0 : 1;
uint64_t u64_bit_1 : 1;
uint64_t u64_bit_2 : 1;
uint64_t u64_bit_3 : 1;
uint64_t u64_bit_4 : 1;
uint64_t u64_bit_5 : 1;
uint64_t u64_bit_6 : 1;
uint64_t u64_bit_7 : 1;
uint64_t u64_bit_8 : 1;
uint64_t u64_bit_9 : 1;
uint64_t u64_bit_10 : 1;
uint64_t u64_bit_11 : 1;
uint64_t u64_bit_12 : 1;
uint64_t u64_bit_13 : 1;
uint64_t u64_bit_14 : 1;
uint64_t u64_bit_15 : 1;
uint64_t u64_bit_16 : 1;
uint64_t u64_bit_17 : 1;
uint64_t u64_bit_18 : 1;
uint64_t u64_bit_19 : 1;
uint64_t u64_bit_20 : 1;
uint64_t u64_bit_21 : 1;
uint64_t u64_bit_22 : 1;
uint64_t u64_bit_23 : 1;
uint64_t u64_bit_24 : 1;
uint64_t u64_bit_25 : 1;
uint64_t u64_bit_26 : 1;
uint64_t u64_bit_27 : 1;
uint64_t u64_bit_28 : 1;
uint64_t u64_bit_29 : 1;
uint64_t u64_bit_30 : 1;
uint64_t u64_bit_31 : 1;
uint64_t u64_bit_32 : 1;
uint64_t u64_bit_33 : 1;
uint64_t u64_bit_34 : 1;
uint64_t u64_bit_35 : 1;
uint64_t u64_bit_36 : 1;
uint64_t u64_bit_37 : 1;
uint64_t u64_bit_38 : 1;
uint64_t u64_bit_39 : 1;
uint64_t u64_bit_40 : 1;
uint64_t u64_bit_41 : 1;
uint64_t u64_bit_42 : 1;
uint64_t u64_bit_43 : 1;
uint64_t u64_bit_44 : 1;
uint64_t u64_bit_45 : 1;
uint64_t u64_bit_46 : 1;
uint64_t u64_bit_47 : 1;
uint64_t u64_bit_48 : 1;
uint64_t u64_bit_49 : 1;
uint64_t u64_bit_50 : 1;
uint64_t u64_bit_51 : 1;
uint64_t u64_bit_52 : 1;
uint64_t u64_bit_53 : 1;
uint64_t u64_bit_54 : 1;
uint64_t u64_bit_55 : 1;
uint64_t u64_bit_56 : 1;
uint64_t u64_bit_57 : 1;
uint64_t u64_bit_58 : 1;
uint64_t u64_bit_59 : 1;
uint64_t u64_bit_60 : 1;
uint64_t u64_bit_61 : 1;
uint64_t u64_bit_62 : 1;
uint64_t u64_bit_63 : 1;
};
};
};
struct bitfield masks[] = {
{.u64_bit_0 = 1},
{.u64_bit_1 = 1},
{.u64_bit_2 = 1},
{.u64_bit_3 = 1},
{.u64_bit_4 = 1},
{.u64_bit_5 = 1},
{.u64_bit_6 = 1},
{.u64_bit_7 = 1},
{.u64_bit_8 = 1},
{.u64_bit_9 = 1},
{.u64_bit_10 = 1},
{.u64_bit_11 = 1},
{.u64_bit_12 = 1},
{.u64_bit_13 = 1},
{.u64_bit_14 = 1},
{.u64_bit_15 = 1},
{.u64_bit_16 = 1},
{.u64_bit_17 = 1},
{.u64_bit_18 = 1},
{.u64_bit_19 = 1},
{.u64_bit_20 = 1},
{.u64_bit_21 = 1},
{.u64_bit_22 = 1},
{.u64_bit_23 = 1},
{.u64_bit_24 = 1},
{.u64_bit_25 = 1},
{.u64_bit_26 = 1},
{.u64_bit_27 = 1},
{.u64_bit_28 = 1},
{.u64_bit_29 = 1},
{.u64_bit_30 = 1},
{.u64_bit_31 = 1},
{.u64_bit_32 = 1},
{.u64_bit_33 = 1},
{.u64_bit_34 = 1},
{.u64_bit_35 = 1},
{.u64_bit_36 = 1},
{.u64_bit_37 = 1},
{.u64_bit_38 = 1},
{.u64_bit_39 = 1},
{.u64_bit_40 = 1},
{.u64_bit_41 = 1},
{.u64_bit_42 = 1},
{.u64_bit_43 = 1},
{.u64_bit_44 = 1},
{.u64_bit_45 = 1},
{.u64_bit_46 = 1},
{.u64_bit_47 = 1},
{.u64_bit_48 = 1},
{.u64_bit_49 = 1},
{.u64_bit_50 = 1},
{.u64_bit_51 = 1},
{.u64_bit_52 = 1},
{.u64_bit_53 = 1},
{.u64_bit_54 = 1},
{.u64_bit_55 = 1},
{.u64_bit_56 = 1},
{.u64_bit_57 = 1},
{.u64_bit_58 = 1},
{.u64_bit_59 = 1},
{.u64_bit_60 = 1},
{.u64_bit_61 = 1},
{.u64_bit_62 = 1},
{.u64_bit_63 = 1}
};
int main(int argc, char **argv) {
struct bitfield *mask_ptr = &masks[0];
return mask_ptr->val;
}
`

View File

@@ -1,85 +0,0 @@
#!/usr/bin/env perl
# Copyright 2009 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
use strict;
if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
print STDERR "GOARCH or GOOS not defined in environment\n";
exit 1;
}
# Check that we are using the new build system if we should
if($ENV{'GOLANG_SYS_BUILD'} ne "docker") {
print STDERR "In the new build system, mksysnum should not be called directly.\n";
print STDERR "See README.md\n";
exit 1;
}
my $command = "$0 ". join(' ', @ARGV);
print <<EOF;
// $command
// Code generated by the command above; see README.md. DO NOT EDIT.
// +build $ENV{'GOARCH'},$ENV{'GOOS'}
package unix
const(
EOF
my $offset = 0;
sub fmt {
my ($name, $num) = @_;
if($num > 999){
# ignore deprecated syscalls that are no longer implemented
# https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/include/uapi/asm-generic/unistd.h?id=refs/heads/master#n716
return;
}
$name =~ y/a-z/A-Z/;
$num = $num + $offset;
print " SYS_$name = $num;\n";
}
my $prev;
open(CC, "$ENV{'CC'} -E -dD @ARGV |") || die "can't run $ENV{'CC'}";
while(<CC>){
if(/^#define __NR_Linux\s+([0-9]+)/){
# mips/mips64: extract offset
$offset = $1;
}
elsif(/^#define __NR(\w*)_SYSCALL_BASE\s+([0-9]+)/){
# arm: extract offset
$offset = $1;
}
elsif(/^#define __NR_syscalls\s+/) {
# ignore redefinitions of __NR_syscalls
}
elsif(/^#define __NR_(\w*)Linux_syscalls\s+/) {
# mips/mips64: ignore definitions about the number of syscalls
}
elsif(/^#define __NR_(\w+)\s+([0-9]+)/){
$prev = $2;
fmt($1, $2);
}
elsif(/^#define __NR3264_(\w+)\s+([0-9]+)/){
$prev = $2;
fmt($1, $2);
}
elsif(/^#define __NR_(\w+)\s+\(\w+\+\s*([0-9]+)\)/){
fmt($1, $prev+$2)
}
elsif(/^#define __NR_(\w+)\s+\(__NR_Linux \+ ([0-9]+)/){
fmt($1, $2);
}
elsif(/^#define __NR_(\w+)\s+\(__NR_SYSCALL_BASE \+ ([0-9]+)/){
fmt($1, $2);
}
}
print <<EOF;
)
EOF

File diff suppressed because it is too large Load Diff