Support arm64/aarch64 Linux and Apple Silicon

- remove references to 32 bit macos/darwin
- normalize Redis server file names
- update `build-server-binaries.sh` to statically link openssl on macOS
- correctly detect macOS and Unix architectures

Co-authored-by: Nathan J Mehl <n@oden.io>
This commit is contained in:
Chris Eager 2022-01-11 10:35:00 -08:00 committed by GitHub
parent d8f71db407
commit 445a8e67e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 203 additions and 110 deletions

View File

@ -38,10 +38,11 @@ RedisServer redisServer = new RedisServer("/path/to/your/redis", 6379);
// 2) given os-independent matrix
RedisExecProvider customProvider = RedisExecProvider.defaultProvider()
.override(OS.UNIX, "/path/to/unix/redis")
.override(OS.WINDOWS, Architecture.x86, "/path/to/windows/redis")
.override(OS.Windows, Architecture.x86_64, "/path/to/windows/redis")
.override(OS.MAC_OS_X, Architecture.x86, "/path/to/macosx/redis")
.override(OS.MAC_OS_X, Architecture.x86_64, "/path/to/macosx/redis")
.override(OS.UNIX, Architecture.x86_64, "/path/to/unix/redis.x86_64")
.override(OS.UNIX, Architecture.arm64, "/path/to/unix/redis.arm64")
.override(OS.UNIX, Architecture.x86, "/path/to/unix/redis.i386")
.override(OS.MAC_OS_X, Architecture.x86_64, "/path/to/macosx/redis-x86_64")
.override(OS.MAC_OS_X, Architecture.arm64, "/path/to/macosx/redis.arm64")
RedisServer redisServer = new RedisServer(customProvider, 6379);
```
@ -141,10 +142,12 @@ Redis version
By default, RedisServer runs an OS-specific executable enclosed in in the `embedded-redis` jar. The jar includes:
- Redis 6.2.6 for Linux/Unix (amd64 and x86)
- Redis 6.2.6 for macOS (amd64)
- Redis 6.2.6 for Linux/Unix (i386, x86_64 and arm64)
- Redis 6.2.6 for macOS (x86_64 and arm64e AKA Apple Silicon)
The enclosed binaries are built from source from the [`6.2.6` tag](https://github.com/redis/redis/releases/tag/6.2.6) in the official Redis repository. The Linux binaries are statically-linked amd64 and x86 executables built using the `build-server-binaries.sh` script included in this repository at `/src/main/docker`. The macOS binaries are built according to the [instructions in the README](https://github.com/redis/redis/blob/4930d19e70c391750479951022e207e19111eb55/README.md#building-redis). Windows binaries are not included because Windows is not officially supported by Redis.
The enclosed binaries are built from source from the [`6.2.6` tag](https://github.com/redis/redis/releases/tag/6.2.6) in the official Redis repository. The Linux and Darwin/macOS binaries are statically-linked amd64 and x86 executables built using the [build-server-binaries.sh](src/main/docker/build-server-binaries.sh) script included in this repository at `/src/main/docker`. Windows binaries are not included because Windows is not officially supported by Redis.
Note: the `build-server-binaries.sh` script attempts to build all of the above noted OS and architectures, which means that it expects the local Docker daemon to support all of them. Docker Desktop on macOS and Windows supports multi-arch builds out of the box; Docker on Linux may require [additional configuration](https://docs.docker.com/buildx/working-with-buildx/).
Callers may provide a path to a specific `redis-server` executable if needed.
@ -182,8 +185,9 @@ Changelog
==============
### 0.8.2
* Compiled Redis servers with TLS support
* Updated to Redis 6.2.6
* Added native support for Apple Silicon (darwin/arm64) and Linux aarch64
* Compiled Redis servers with TLS support
### 0.8.1

View File

@ -95,6 +95,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>8</source>
<target>8</target>

2
src/main/docker/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
redis-server-*
redis-*

View File

@ -0,0 +1,18 @@
ARG ALPINE_VERSION=3.12
FROM alpine:${ALPINE_VERSION}
RUN apk add --no-cache gcc musl-dev libressl-dev make pkgconfig linux-headers
WORKDIR /build
ARG REDIS_VERSION
ENV REDIS_VERSION=${REDIS_VERSION}
COPY redis-${REDIS_VERSION}.tar.gz /redis-${REDIS_VERSION}.tar.gz
RUN ls -l /
ARG ARCH
RUN tar zxf /redis-${REDIS_VERSION}.tar.gz && \
cd redis-${REDIS_VERSION} && \
make BUILD_TLS='yes' CC='gcc -static' LDFLAGS='-s' MALLOC='libc' && \
mv src/redis-server /build/redis-server-${REDIS_VERSION}-linux-${ARCH}
CMD [ "/bin/sh" ]

View File

@ -1,10 +0,0 @@
FROM alpine:3.12
ARG REDIS_VERSION
RUN apk add --no-cache gcc musl-dev libressl-dev wget make pkgconfig linux-headers; \
mkdir /redis-server-binaries; \
wget https://download.redis.io/releases/redis-${REDIS_VERSION}.tar.gz; \
tar xfz redis-${REDIS_VERSION}.tar.gz;
CMD [ "/bin/sh" ]

View File

@ -1,16 +1,115 @@
#!/bin/bash
set -e
REDIS_VERSION=6.2.6
REDIS_TARBALL="redis-${REDIS_VERSION}.tar.gz"
REDIS_URL="https://download.redis.io/releases/${REDIS_TARBALL}"
docker build --build-arg REDIS_VERSION=${REDIS_VERSION} -t redis-server-builder-amd64 -f ./build-server-amd64.docker .
docker build --build-arg REDIS_VERSION=${REDIS_VERSION} -t redis-server-builder-x86 -f ./build-server-x86.docker .
echo $ARCH
docker run -it --rm \
-v "$(pwd)/":/redis-server-binaries \
redis-server-builder-amd64 \
sh -c "cd redis-${REDIS_VERSION}; make BUILD_TLS='yes' CC='gcc -static' LDFLAGS='-s' MALLOC='libc'; cp src/redis-server /redis-server-binaries/redis-server-${REDIS_VERSION}"
function copy_openssl_and_remove_dylibs() {
# To make macOS builds more portable, we want to statically link OpenSSL,
# which is not straightforward. To force static compilation, we copy
# the openssl libraries and remove dylibs, forcing static linking
OPENSSL_HOME="${1}"
ARCH=$2
OPENSSL_HOME_COPY="${3}/${ARCH}"
if [ -d "${OPENSSL_HOME}" ]; then
echo "*** Copying openssl libraries for static linking"
cp -RL "${OPENSSL_HOME}" "${OPENSSL_HOME_COPY}"
rm -f "${OPENSSL_HOME_COPY}"/lib/*.dylib
else
echo "*** WARNING: could not find openssl libraries at ${OPENSSL_HOME}; this build will almost certainly fail"
fi
}
docker run -it --rm \
-v "$(pwd)/":/redis-server-binaries \
redis-server-builder-x86 \
sh -c "cd redis-${REDIS_VERSION}; make BUILD_TLS='yes' CC='gcc -static' CFLAGS='-m32' LDFLAGS='-m32 -s' MALLOC='libc'; cp src/redis-server /redis-server-binaries/redis-server-${REDIS_VERSION}-32"
if [ "$(dirname ${0})" != "." ]; then
echo "This script must be run from $(dirname ${0}). \`cd\` there and run again"
exit 1
fi
if ! [ -f "${REDIS_TARBALL}" ]; then
curl -o "${REDIS_TARBALL}" "${REDIS_URL}"
fi
all_linux=0
if command -pv docker 2>/dev/null; then
for arch in x86_64 arm64 i386; do
echo "*** Building redis version ${REDIS_VERSION} for linux-${arch}"
set +e
docker build \
"--platform=linux/${arch}" \
--build-arg "REDIS_VERSION=${REDIS_VERSION}" \
--build-arg "ARCH=${arch}" \
-t "redis-server-builder-${arch}" \
.
if [[ $? -ne 0 ]]; then
echo "*** ERROR: could not build for linux-${arch}"
continue
fi
set -e
docker run -it --rm \
"--platform=linux/${arch}" \
-v "$(pwd)/":/mnt \
"redis-server-builder-${arch}" \
sh -c "cp /build/redis-server-${REDIS_VERSION}-linux-${arch} /mnt"
((all_linux+=1))
done
else
echo "*** WARNING: No docker command found. Cannot build for linux."
fi
if [[ "${all_linux}" -lt 3 ]]; then
echo "*** WARNING: was not able to build for all linux arches; see above for errors"
fi
if [[ "$(uname -s)" == "Darwin" ]]; then
tar zxf "${REDIS_TARBALL}"
cd "redis-${REDIS_VERSION}"
# temporary directory for openssl libraries for static linking.
# assumes standard Homebrew openssl install:
# - arm64e at /opt/homebrew/opt/openssl
# - x86_64 at /usr/local/opt/openssl
OPENSSL_TEMP=$(mktemp -d /tmp/embedded-redis-darwin-openssl.XXXXX)
# build for arm64 on apple silicon
if arch -arm64e true 2>/dev/null; then
copy_openssl_and_remove_dylibs /opt/homebrew/opt/openssl arm64e "${OPENSSL_TEMP}"
echo "*** Building redis version ${REDIS_VERSION} for darwin-arm64e (apple silicon)"
make clean
arch -arm64e make -j3 BUILD_TLS=yes OPENSSL_PREFIX="$OPENSSL_TEMP/arm64e"
mv src/redis-server "../redis-server-${REDIS_VERSION}-darwin-arm64"
else
echo "*** WARNING: could not build for darwin-arm64e; you probably want to do this on an apple silicon device"
fi
# build for x86_64 if we're on apple silicon or a recent macos on x86_64
if arch -x86_64 true 2>/dev/null; then
copy_openssl_and_remove_dylibs /usr/local/opt/openssl x86_64 "${OPENSSL_TEMP}"
echo "*** Building redis version ${REDIS_VERSION} for darwin-x86_64"
make clean
arch -x86_64 make -j3 BUILD_TLS=yes OPENSSL_PREFIX="$OPENSSL_TEMP/x86_64"
mv src/redis-server "../redis-server-${REDIS_VERSION}-darwin-x86_64"
else
echo "*** WARNING: you are on a version of macos that lacks /usr/bin/arch, you probably do not want this"
exit 1
fi
cd ..
else
echo "*** WARNING: Cannot build for macos/darwin on a $(uname -s) host"
fi
ls -l redis-server-*
echo "*** Moving built binaries to ../resources; you need to handle the rest yourself"
mv redis-server-* ../resources/

View File

@ -1,10 +0,0 @@
FROM i386/alpine:3.12
ARG REDIS_VERSION
RUN apk add --no-cache gcc musl-dev libressl-dev wget make pkgconfig linux-headers; \
mkdir /redis-server-binaries; \
wget https://download.redis.io/releases/redis-${REDIS_VERSION}.tar.gz; \
tar xfz redis-${REDIS_VERSION}.tar.gz;
CMD [ "/bin/sh" ]

View File

@ -76,7 +76,7 @@ abstract class AbstractRedisInstance implements Redis {
}
} while (!outputLine.matches(redisReadyPattern()));
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(reader, null);
}
}
@ -127,7 +127,7 @@ abstract class AbstractRedisInstance implements Redis {
try {
readLines();
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(reader, null);
}
}

View File

@ -15,6 +15,8 @@ public class RedisExecProvider {
private final Map<OsArchitecture, String> executables = Maps.newHashMap();
public static final String redisVersion = "6.2.6";
public static RedisExecProvider defaultProvider() {
return new RedisExecProvider();
}
@ -24,11 +26,12 @@ public class RedisExecProvider {
}
private void initExecutables() {
executables.put(OsArchitecture.UNIX_x86, "redis-server-6.2.6-32");
executables.put(OsArchitecture.UNIX_x86_64, "redis-server-6.2.6");
executables.put(OsArchitecture.UNIX_x86, "redis-server-" + redisVersion + "-linux-i386");
executables.put(OsArchitecture.UNIX_x86_64, "redis-server-" + redisVersion + "-linux-x86_64");
executables.put(OsArchitecture.UNIX_arm64, "redis-server-" + redisVersion + "-linux-arm64");
executables.put(OsArchitecture.MAC_OS_X_x86, "redis-server-6.2.6.app");
executables.put(OsArchitecture.MAC_OS_X_x86_64, "redis-server-6.2.6.app");
executables.put(OsArchitecture.MAC_OS_X_x86_64, "redis-server-" + redisVersion + "-darwin-x86_64");
executables.put(OsArchitecture.MAC_OS_X_arm64, "redis-server-" + redisVersion + "-darwin-arm64");
}
public RedisExecProvider override(OS os, String executable) {

View File

@ -7,6 +7,7 @@ import redis.embedded.exceptions.RedisBuildingException;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@ -139,7 +140,7 @@ public class RedisSentinelBuilder {
File redisConfigFile = File.createTempFile(resolveConfigName(), ".conf");
redisConfigFile.deleteOnExit();
Files.write(configString, redisConfigFile, Charset.forName("UTF-8"));
Files.asCharSink(redisConfigFile, StandardCharsets.UTF_8).write(configString);
sentinelConf = redisConfigFile.getAbsolutePath();
}

View File

@ -8,6 +8,7 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@ -103,7 +104,7 @@ public class RedisServerBuilder {
if (redisConf == null && redisConfigBuilder != null) {
File redisConfigFile = File.createTempFile(resolveConfigName(), ".conf");
redisConfigFile.deleteOnExit();
Files.write(redisConfigBuilder.toString(), redisConfigFile, Charset.forName("UTF-8"));
Files.asCharSink(redisConfigFile, StandardCharsets.UTF_8).write(redisConfigBuilder.toString());
redisConf = redisConfigFile.getAbsolutePath();
}

View File

@ -2,5 +2,6 @@ package redis.embedded.util;
public enum Architecture {
x86,
x86_64
x86_64,
arm64
}

View File

@ -47,58 +47,46 @@ public class OSDetector {
}
private static Architecture getUnixArchitecture() {
BufferedReader input = null;
try {
String line;
Process proc = Runtime.getRuntime().exec("uname -m");
input = new BufferedReader(new InputStreamReader(proc.getInputStream()));
while ((line = input.readLine()) != null) {
if (line.length() > 0) {
if (line.contains("64")) {
try (BufferedReader input = new BufferedReader(new InputStreamReader(proc.getInputStream()))) {
String machine = input.readLine();
switch (machine) {
case "aarch64":
return Architecture.arm64;
case "x86_64":
return Architecture.x86_64;
}
default:
throw new OsDetectionException("unsupported architecture: " + machine);
}
}
} catch (Exception e) {
throw new OsDetectionException(e);
} finally {
try {
if (input != null) {
input.close();
}
} catch (Exception ignored) {
// ignore
if (e instanceof OsDetectionException) {
throw (OsDetectionException)e;
}
throw new OsDetectionException(e);
}
return Architecture.x86;
}
private static Architecture getMacOSXArchitecture() {
BufferedReader input = null;
try {
String line;
Process proc = Runtime.getRuntime().exec("sysctl hw");
input = new BufferedReader(new InputStreamReader(proc.getInputStream()));
while ((line = input.readLine()) != null) {
if (line.length() > 0) {
if ((line.contains("cpu64bit_capable")) && (line.trim().endsWith("1"))) {
Process proc = Runtime.getRuntime().exec("uname -m");
try (BufferedReader input = new BufferedReader(new InputStreamReader(proc.getInputStream()))) {
String machine = input.readLine();
switch (machine) {
case "arm64":
return Architecture.arm64;
case "x86_64":
return Architecture.x86_64;
}
default:
throw new OsDetectionException("unsupported architecture: " + machine);
}
}
} catch (Exception e) {
throw new OsDetectionException(e);
} finally {
try {
if (input != null) {
input.close();
}
} catch (Exception ignored) {
// ignore
if (e instanceof OsDetectionException) {
throw (OsDetectionException)e;
}
throw new OsDetectionException(e);
}
return Architecture.x86;
}
}

View File

@ -6,9 +6,10 @@ public class OsArchitecture {
public static final OsArchitecture UNIX_x86 = new OsArchitecture(OS.UNIX, Architecture.x86);
public static final OsArchitecture UNIX_x86_64 = new OsArchitecture(OS.UNIX, Architecture.x86_64);
public static final OsArchitecture UNIX_arm64 = new OsArchitecture(OS.UNIX, Architecture.arm64);
public static final OsArchitecture MAC_OS_X_x86 = new OsArchitecture(OS.MAC_OS_X, Architecture.x86);
public static final OsArchitecture MAC_OS_X_x86_64 = new OsArchitecture(OS.MAC_OS_X, Architecture.x86_64);
public static final OsArchitecture MAC_OS_X_arm64 = new OsArchitecture(OS.MAC_OS_X, Architecture.arm64);
private final OS os;
private final Architecture arch;

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -1,12 +1,14 @@
package redis.embedded;
import com.google.common.collect.Sets;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisSentinelPool;
import redis.embedded.util.JedisUtil;
import java.io.Closeable;
import java.net.Inet4Address;
import java.util.Arrays;
import java.util.List;
@ -118,8 +120,7 @@ public class RedisClusterTest {
pool = new JedisSentinelPool("ourmaster", Sets.newHashSet("localhost:26379"));
jedis = testPool(pool);
} finally {
if (jedis != null)
pool.returnResource(jedis);
closeQuietly(jedis, pool);
cluster.stop();
}
}
@ -137,8 +138,7 @@ public class RedisClusterTest {
pool = new JedisSentinelPool("ourmaster", Sets.newHashSet("localhost:26379"));
jedis = testPool(pool);
} finally {
if (jedis != null)
pool.returnResource(jedis);
closeQuietly(jedis, pool);
cluster.stop();
}
}
@ -156,8 +156,7 @@ public class RedisClusterTest {
pool = new JedisSentinelPool("ourmaster", Sets.newHashSet("localhost:26379"));
jedis = testPool(pool);
} finally {
if (jedis != null)
pool.returnResource(jedis);
closeQuietly(jedis, pool);
cluster.stop();
}
}
@ -175,8 +174,7 @@ public class RedisClusterTest {
pool = new JedisSentinelPool("ourmaster", Sets.newHashSet("localhost:26379", "localhost:26380"));
jedis = testPool(pool);
} finally {
if (jedis != null)
pool.returnResource(jedis);
closeQuietly(jedis, pool);
cluster.stop();
}
}
@ -196,8 +194,7 @@ public class RedisClusterTest {
pool = new JedisSentinelPool("ourmaster", sentinelHosts);
jedis = testPool(pool);
} finally {
if (jedis != null)
pool.returnResource(jedis);
closeQuietly(jedis, pool);
cluster.stop();
}
}
@ -230,12 +227,7 @@ public class RedisClusterTest {
jedis2 = testPool(pool2);
jedis3 = testPool(pool3);
} finally {
if (jedis1 != null)
pool1.returnResource(jedis1);
if (jedis2 != null)
pool2.returnResource(jedis2);
if (jedis3 != null)
pool3.returnResource(jedis3);
closeQuietly(jedis1, pool1, jedis2, pool2, jedis3, pool3);
cluster.stop();
}
}
@ -269,12 +261,7 @@ public class RedisClusterTest {
jedis2 = testPool(pool2);
jedis3 = testPool(pool3);
} finally {
if (jedis1 != null)
pool1.returnResource(jedis1);
if (jedis2 != null)
pool2.returnResource(jedis2);
if (jedis3 != null)
pool3.returnResource(jedis3);
closeQuietly(jedis1, pool1, jedis2, pool2, jedis3, pool3);
cluster.stop();
}
}
@ -290,4 +277,11 @@ public class RedisClusterTest {
assertEquals(null, jedis.mget("xyz").get(0));
return jedis;
}
private void closeQuietly(Closeable... closeables) {
for (Closeable closeable : closeables) {
IOUtils.closeQuietly(closeable, null);
}
}
}

View File

@ -97,9 +97,11 @@ public class RedisServerTest {
@Test
public void shouldOverrideDefaultExecutable() throws Exception {
RedisExecProvider customProvider = RedisExecProvider.defaultProvider()
.override(OS.UNIX, Architecture.x86, Resources.getResource("redis-server-6.2.6-32").getFile())
.override(OS.UNIX, Architecture.x86_64, Resources.getResource("redis-server-6.2.6").getFile())
.override(OS.MAC_OS_X, Resources.getResource("redis-server-6.2.6").getFile());
.override(OS.UNIX, Architecture.x86, Resources.getResource("redis-server-" + RedisExecProvider.redisVersion + "-linux-i386").getFile())
.override(OS.UNIX, Architecture.x86_64, Resources.getResource("redis-server-" + RedisExecProvider.redisVersion + "-linux-x86_64").getFile())
.override(OS.UNIX, Architecture.arm64, Resources.getResource("redis-server-" + RedisExecProvider.redisVersion + "-linux-arm64").getFile())
.override(OS.MAC_OS_X, Architecture.x86_64, Resources.getResource("redis-server-" + RedisExecProvider.redisVersion + "-darwin-x86_64").getFile())
.override(OS.MAC_OS_X, Architecture.arm64, Resources.getResource("redis-server-" + RedisExecProvider.redisVersion + "-darwin-arm64").getFile());
redisServer = new RedisServerBuilder()
.redisExecProvider(customProvider)
@ -110,8 +112,6 @@ public class RedisServerTest {
public void shouldFailWhenBadExecutableGiven() throws Exception {
RedisExecProvider buggyProvider = RedisExecProvider.defaultProvider()
.override(OS.UNIX, "some")
.override(OS.WINDOWS, Architecture.x86, "some")
.override(OS.WINDOWS, Architecture.x86_64, "some")
.override(OS.MAC_OS_X, "some");
redisServer = new RedisServerBuilder()