chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:05 +08:00
commit 4f3b7da785
7394 changed files with 2005594 additions and 0 deletions
+69
View File
@@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ /* ******************************************************************************
~ *
~ *
~ * This program and the accompanying materials are made available under the
~ * terms of the Apache License, Version 2.0 which is available at
~ * https://www.apache.org/licenses/LICENSE-2.0.
~ *
~ * See the NOTICE file distributed with this work for additional
~ * information regarding copyright ownership.
~ * Unless required by applicable law or agreed to in writing, software
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
~ * License for the specific language governing permissions and limitations
~ * under the License.
~ *
~ * SPDX-License-Identifier: Apache-2.0
~ ******************************************************************************/
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>python4j-parent</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>python4j-numpy</artifactId>
<properties>
<module.name>python4j.numpy</module.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.moditect</groupId>
<artifactId>moditect-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>numpy-platform</artifactId>
<version>${numpy.javacpp.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>nd4j-native-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.deeplearning4j</groupId>
<artifactId>python4j-core</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,336 @@
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.python4j.numpy;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.bytedeco.cpython.PyObject;
import org.bytedeco.cpython.PyTypeObject;
import org.bytedeco.javacpp.Pointer;
import org.bytedeco.javacpp.SizeTPointer;
import org.bytedeco.numpy.PyArrayObject;
import org.bytedeco.numpy.global.numpy;
import org.nd4j.linalg.api.buffer.DataBuffer;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.concurrency.AffinityManager;
import org.nd4j.linalg.api.memory.MemoryWorkspace;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.shape.Shape;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.nativeblas.NativeOpsHolder;
import org.nd4j.python4j.*;
import java.io.File;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.bytedeco.cpython.global.python.*;
import static org.bytedeco.numpy.global.numpy.*;
@Slf4j
public class NumpyArray extends PythonType<INDArray> {
public static final NumpyArray INSTANCE;
private static final AtomicBoolean init = new AtomicBoolean(false);
private static final Map<String, DataBuffer> cache = new HashMap<>();
public final static String IMPORT_NUMPY_ARRAY = "org.eclipse.python4j.numpyimport";
public final static String ADD_JAVACPP_NUMPY_TO_PATH = "org.eclipse.python4j.numpyimport";
public final static String DEFAULT_IMPORT_NUMPY_ARRAY = "true";
public final static String DEFAULT_ADD_JAVACPP_NUMPY_TO_PATH = "true";
static {
new PythonExecutioner();
INSTANCE = new NumpyArray();
INSTANCE.init();
}
@Override
public File[] packages(){
try{
return new File[]{numpy.cachePackage()};
}catch(Exception e){
throw new PythonException(e);
}
}
@SneakyThrows
public synchronized void init() {
if (init.get()) return;
init.set(true);
if(Boolean.parseBoolean(System.getProperty(IMPORT_NUMPY_ARRAY,DEFAULT_IMPORT_NUMPY_ARRAY))) {
//See: https://numpy.org/doc/1.17/reference/c-api.array.html#importing-the-api
//DO NOT REMOVE
if(Boolean.parseBoolean(System.getProperty(ADD_JAVACPP_NUMPY_TO_PATH,DEFAULT_ADD_JAVACPP_NUMPY_TO_PATH))) {
for(File packageDir : numpy.cachePackages()) {
PyObject pythonPath = PyUnicode_FromString(packageDir.getAbsolutePath());
PySys_SetObject("path", pythonPath);
Py_DecRef(pythonPath);
}
}
//ensure python doesn't get initialized twice, this call is needed before numpy import array
PythonConstants.setInitializePython(false);
Py_Initialize();
int err = numpy._import_array();
if (err < 0){
System.out.println("Numpy import failed!");
throw new PythonException("Numpy import failed!");
}
}
if (PythonGIL.locked()) {
throw new PythonException("Can not initialize numpy - GIL already acquired.");
}
}
public NumpyArray() {
super("numpy.ndarray", INDArray.class);
}
@Override
public INDArray toJava(PythonObject pythonObject) {
log.debug("Converting PythonObject to INDArray...");
PyObject np = PyImport_ImportModule("numpy");
PyObject ndarray = PyObject_GetAttrString(np, "ndarray");
if (PyObject_IsInstance(pythonObject.getNativePythonObject(), ndarray) != 1) {
Py_DecRef(ndarray);
Py_DecRef(np);
throw new PythonException("Object is not a numpy array! Use Python.ndarray() to convert object to a numpy array.");
}
Py_DecRef(ndarray);
Py_DecRef(np);
PyArrayObject npArr = new PyArrayObject(pythonObject.getNativePythonObject());
long[] shape = new long[PyArray_NDIM(npArr)];
SizeTPointer shapePtr = PyArray_SHAPE(npArr);
if (shapePtr != null)
shapePtr.get(shape, 0, shape.length);
long[] strides = new long[shape.length];
SizeTPointer stridesPtr = PyArray_STRIDES(npArr);
if (stridesPtr != null)
stridesPtr.get(strides, 0, strides.length);
int npdtype = PyArray_TYPE(npArr);
DataType dtype;
switch (npdtype) {
case NPY_DOUBLE:
dtype = DataType.DOUBLE;
break;
case NPY_FLOAT:
dtype = DataType.FLOAT;
break;
case NPY_SHORT:
dtype = DataType.SHORT;
break;
case NPY_INT:
dtype = DataType.INT32;
break;
case NPY_LONG:
dtype = DataType.INT64;
break;
case NPY_UINT:
dtype = DataType.UINT32;
break;
case NPY_BYTE:
dtype = DataType.INT8;
break;
case NPY_UBYTE:
dtype = DataType.UINT8;
break;
case NPY_BOOL:
dtype = DataType.BOOL;
break;
case NPY_HALF:
dtype = DataType.FLOAT16;
break;
case NPY_LONGLONG:
dtype = DataType.INT64;
break;
case NPY_USHORT:
dtype = DataType.UINT16;
break;
case NPY_ULONG:
case NPY_ULONGLONG:
dtype = DataType.UINT64;
break;
default:
throw new PythonException("Unsupported array data type: " + npdtype);
}
long size = 1;
for (int i = 0; i < shape.length; size *= shape[i++]) ;
INDArray ret;
long address = PyArray_DATA(npArr).address();
String key = address + "_" + size + "_" + dtype;
DataBuffer buff = cache.get(key);
if (buff == null) {
try (MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
Pointer ptr = Nd4j.getNativeOps().pointerForAddress(address);
ptr = ptr.limit(size);
ptr = ptr.capacity(size);
buff = Nd4j.createBuffer(ptr, size, dtype);
cache.put(key, buff);
}
}
int elemSize = buff.getElementSize();
long[] nd4jStrides = new long[strides.length];
for (int i = 0; i < strides.length; i++) {
nd4jStrides[i] = strides[i] / elemSize;
}
ret = Nd4j.create(buff, shape, nd4jStrides, 0, Shape.getOrder(shape, nd4jStrides, 1), dtype);
Nd4j.getAffinityManager().tagLocation(ret, AffinityManager.Location.HOST);
log.debug("Done creating numpy array.");
return ret;
}
@Override
public PythonObject toPython(INDArray indArray) {
log.debug("Converting INDArray to PythonObject...");
DataType dataType = indArray.dataType();
DataBuffer buff = indArray.data();
String key = buff.pointer().address() + "_" + buff.length() + "_" + dataType;
cache.put(key, buff);
int numpyType;
String ctype;
switch (dataType) {
case DOUBLE:
numpyType = NPY_DOUBLE;
ctype = "c_double";
break;
case FLOAT:
case BFLOAT16:
numpyType = NPY_FLOAT;
ctype = "c_float";
break;
case SHORT:
numpyType = NPY_SHORT;
ctype = "c_short";
break;
case INT:
numpyType = NPY_INT;
ctype = "c_int";
break;
case LONG:
numpyType = NPY_INT64;
ctype = "c_int64";
break;
case UINT16:
numpyType = NPY_USHORT;
ctype = "c_uint16";
break;
case UINT32:
numpyType = NPY_UINT;
ctype = "c_uint";
break;
case UINT64:
numpyType = NPY_UINT64;
ctype = "c_uint64";
break;
case BOOL:
numpyType = NPY_BOOL;
ctype = "c_bool";
break;
case BYTE:
numpyType = NPY_BYTE;
ctype = "c_byte";
break;
case UBYTE:
numpyType = NPY_UBYTE;
ctype = "c_ubyte";
break;
case HALF:
numpyType = NPY_HALF;
ctype = "c_short";
break;
default:
throw new RuntimeException("Unsupported dtype: " + dataType);
}
long[] shape = indArray.shape();
INDArray inputArray = indArray;
if (dataType == DataType.BFLOAT16) {
log.warn("Creating copy of array as bfloat16 is not supported by numpy.");
inputArray = indArray.castTo(DataType.FLOAT);
}
//Sync to host memory in the case of CUDA, before passing the host memory pointer to Python
Nd4j.getAffinityManager().ensureLocation(inputArray, AffinityManager.Location.HOST);
// PyArray_Type() call causes jvm crash in linux cpu if GIL is acquired by non main thread.
// Using Interpreter for now:
//likely embedded in python, always use this method instead
if(!PythonConstants.releaseGilAutomatically() || PythonConstants.createNpyViaPython()) {
try(PythonContextManager.Context context = new PythonContextManager.Context("__np_array_converter")){
log.debug("Stringing exec...");
String code = "import ctypes\nimport numpy as np\n" +
"cArr = (ctypes." + ctype + "*" + indArray.length() + ")"+
".from_address(" + indArray.data().pointer().address() + ")\n"+
"npArr = np.frombuffer(cArr, dtype=" + ((numpyType == NPY_HALF) ? "'half'" : "ctypes." + ctype)+
").reshape(" + Arrays.toString(indArray.shape()) + ")";
PythonExecutioner.exec(code);
log.debug("exec done.");
PythonObject ret = PythonExecutioner.getVariable("npArr");
Py_IncRef(ret.getNativePythonObject());
return ret;
}
} else {
log.debug("NUMPY: PyArray_Type()");
PyTypeObject pyTypeObject = PyArray_Type();
log.debug("NUMPY: PyArray_New()");
PyObject npArr = PyArray_New(pyTypeObject, shape.length, new SizeTPointer(shape),
numpyType, null,
inputArray.data().addressPointer(),
0, NPY_ARRAY_CARRAY, null);
log.debug("Created numpy array.");
return new PythonObject(npArr);
}
}
@Override
public boolean accepts(Object javaObject) {
return javaObject instanceof INDArray;
}
@Override
public INDArray adapt(Object javaObject) {
if (javaObject instanceof INDArray) {
return (INDArray) javaObject;
}
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to INDArray");
}
}
@@ -0,0 +1,12 @@
open module python4j.numpy {
requires lombok;
requires nd4j.cpu.api;
requires org.bytedeco.cpython;
requires org.bytedeco.javacpp;
requires org.bytedeco.numpy;
requires slf4j.api;
requires nd4j.api;
requires python4j.core;
exports org.nd4j.python4j.numpy;
provides org.nd4j.python4j.PythonType with org.nd4j.python4j.numpy.NumpyArray;
}
@@ -0,0 +1,217 @@
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
#
# /* ******************************************************************************
# * Copyright (c) 2021 Deeplearning4j Contributors
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
org.nd4j.python4j.numpy.NumpyArray