chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,101 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>deeplearning4j</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>python4j-parent</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<modules>
|
||||
<module>python4j-core</module>
|
||||
<module>python4j-numpy</module>
|
||||
</modules>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>netty-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<!-- Redirect jackson to slf4j. -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>log4j-over-slf4j</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>${commons-io.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
</plugin>
|
||||
<!-- Configuration for git-commit-id plugin - used with ND4J version check functionality -->
|
||||
<plugin>
|
||||
<groupId>pl.project13.maven</groupId>
|
||||
<artifactId>git-commit-id-plugin</artifactId>
|
||||
</plugin>
|
||||
<!-- Add generated git.properties files resource directory, for output of git-commit-id plugin -->
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>build-helper-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>net.revelc.code.formatter</groupId>
|
||||
<artifactId>formatter-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<directories>
|
||||
<directory>python4j-core</directory>
|
||||
<directory>python4j-numpy</directory>
|
||||
</directories>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,95 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>python4j-parent</artifactId>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>python4j-core</artifactId>
|
||||
|
||||
<properties>
|
||||
<module.name>python4j.core</module.name>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.moditect</groupId>
|
||||
<artifactId>moditect-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20190722</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.bytedeco</groupId>
|
||||
<artifactId>cpython-platform</artifactId>
|
||||
<version>${cpython-platform.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.bytedeco</groupId>
|
||||
<artifactId>javacpp-platform</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>netty-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.nd4j.shade.guava.util.concurrent.CycleDetectingLockFactory;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Python GIL holder based on:
|
||||
* https://github.com/invesdwin/invesdwin-context-python/blob/master/invesdwin-context-python-parent/invesdwin-context-python-runtime-python4j/src/main/java/de/invesdwin/context/python/runtime/python4j/internal/GilLock.java
|
||||
* Permission under apache license granted here: https://github.com/eclipse/deeplearning4j/issues/9595
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
public class GILLock implements Lock {
|
||||
|
||||
private static CycleDetectingLockFactory cycleDetectingLockFactory = CycleDetectingLockFactory.newInstance(CycleDetectingLockFactory.Policies.DISABLED);;
|
||||
|
||||
private static final ReentrantLock reentrantLock = cycleDetectingLockFactory.newReentrantLock("python4j_lock");
|
||||
private PythonGIL pythonGIL;
|
||||
private final AtomicInteger lockedCount = new AtomicInteger();
|
||||
|
||||
public void lock() {
|
||||
if(lockedCount.incrementAndGet() == 1) {
|
||||
reentrantLock.lock();
|
||||
pythonGIL = PythonGIL.lock();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lockInterruptibly() throws InterruptedException {
|
||||
lock();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean tryLock() {
|
||||
lock();
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean tryLock(long l, TimeUnit timeUnit) throws InterruptedException {
|
||||
return false;
|
||||
}
|
||||
|
||||
public void unlock() {
|
||||
if(lockedCount.decrementAndGet() == 0) {
|
||||
if (pythonGIL != null)
|
||||
pythonGIL.close();
|
||||
pythonGIL = null;
|
||||
reentrantLock.unlock();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Condition newCondition() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.nd4j.common.primitives.AtomicBoolean;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
|
||||
/**
|
||||
* A port of:
|
||||
* https://github.com/invesdwin/invesdwin-context-python/blob/master/invesdwin-context-python-parent/invesdwin-context-python-runtime-python4j/src/main/java/de/invesdwin/context/python/runtime/python4j/internal/InitializingPythonEngineWrapper.java
|
||||
* Permission given here:
|
||||
* https://github.com/eclipse/deeplearning4j/issues/9595
|
||||
*
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
public class InitializingPythonInterpreter implements PythonInterpreter {
|
||||
|
||||
private static AtomicBoolean initialized = new AtomicBoolean(false);
|
||||
private final PythonInterpreter delegate;
|
||||
|
||||
public InitializingPythonInterpreter() {
|
||||
delegate = UncheckedPythonInterpreter.getInstance();
|
||||
}
|
||||
|
||||
public static void maybeInit() {
|
||||
if (initialized.get()) {
|
||||
return;
|
||||
}
|
||||
synchronized (InitializingPythonInterpreter.class) {
|
||||
if (initialized.get()) {
|
||||
return;
|
||||
}
|
||||
|
||||
UncheckedPythonInterpreter.getInstance().init();
|
||||
initialized.set(true);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getCachedPython(String varName) {
|
||||
return delegate.getCachedPython(varName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getCachedJava(String varName) {
|
||||
return delegate.getCachedJava(varName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Pair<PythonObject, Object> getCachedPythonJava(String varName) {
|
||||
return delegate.getCachedPythonJava(varName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GILLock gilLock() {
|
||||
return delegate.gilLock();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec(String expression) {
|
||||
delegate.exec(expression);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object get(String variable, boolean getNew) {
|
||||
InitializingPythonInterpreter.maybeInit();
|
||||
return delegate.get(variable, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void set(String variable, Object value) {
|
||||
InitializingPythonInterpreter.maybeInit();
|
||||
delegate.set(variable,value);
|
||||
}
|
||||
|
||||
public static PythonInterpreter getInstance() {
|
||||
InitializingPythonInterpreter.maybeInit();
|
||||
return UncheckedPythonInterpreter.getInstance();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
/**
|
||||
* Based on work from:
|
||||
* https://github.com/invesdwin/invesdwin-context-python/blob/master/invesdwin-context-python-parent/invesdwin-context-python-runtime-python4j/src/main/java/de/invesdwin/context/python/runtime/python4j/internal/NonePythonType.java
|
||||
* Permission granted from original author here: https://github.com/eclipse/deeplearning4j/issues/9595
|
||||
*
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
@NotThreadSafe
|
||||
public class NoneType extends PythonType<Void> {
|
||||
|
||||
private static final String ANS = "__ans__";
|
||||
private static final String ANS_EQUALS = ANS + " = ";
|
||||
|
||||
|
||||
public NoneType() {
|
||||
super("NoneType", Void.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accepts(final Object javaObject) {
|
||||
return javaObject == null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void toJava(final PythonObject pythonObject) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(final Void javaObject) {
|
||||
return UncheckedPythonInterpreter.getInstance().newNone();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,615 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
|
||||
|
||||
public class Python {
|
||||
|
||||
static {
|
||||
new PythonExecutioner();
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a python module, similar to python import statement.
|
||||
*
|
||||
* @param moduleName name of the module to be imported
|
||||
* @return reference to the module object
|
||||
*/
|
||||
public static PythonObject importModule(String moduleName) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PythonObject module = new PythonObject(PyImport_ImportModule(moduleName));
|
||||
if (module.isNone()) {
|
||||
throw new PythonException("Error importing module: " + moduleName);
|
||||
}
|
||||
return module;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a builtins attribute
|
||||
*
|
||||
* @param attrName Attribute name
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject attr(String attrName) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject builtins = PyImport_ImportModule("builtins");
|
||||
try {
|
||||
return new PythonObject(PyObject_GetAttrString(builtins, attrName));
|
||||
} finally {
|
||||
Py_DecRef(builtins);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Gets the size of a PythonObject. similar to len() in python.
|
||||
*
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject len(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
long n = PyObject_Size(pythonObject.getNativePythonObject());
|
||||
if (n < 0) {
|
||||
throw new PythonException("Object has no length: " + pythonObject);
|
||||
}
|
||||
return PythonTypes.INT.toPython(n);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the string representation of an object.
|
||||
*
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject str(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
try {
|
||||
return PythonTypes.STR.toPython(pythonObject.toString());
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an empty string
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject str() {
|
||||
PythonGIL.assertThreadSafe();
|
||||
try {
|
||||
return PythonTypes.STR.toPython("");
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the str type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject strType() {
|
||||
return attr("str");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a floating point number from a number or a string.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject float_(PythonObject pythonObject) {
|
||||
return PythonTypes.FLOAT.toPython(PythonTypes.FLOAT.toJava(pythonObject));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reutrns 0.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject float_() {
|
||||
try {
|
||||
return PythonTypes.FLOAT.toPython(0d);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the float type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject floatType() {
|
||||
return attr("float");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Converts a value to a Boolean value i.e., True or False, using the standard truth testing procedure.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bool(PythonObject pythonObject) {
|
||||
return PythonTypes.BOOL.toPython(PythonTypes.BOOL.toJava(pythonObject));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns False.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bool() {
|
||||
return PythonTypes.BOOL.toPython(false);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the bool type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject boolType() {
|
||||
return attr("bool");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an integer from a number or a string.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject int_(PythonObject pythonObject) {
|
||||
return PythonTypes.INT.toPython(PythonTypes.INT.toJava(pythonObject));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns 0
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject int_() {
|
||||
return PythonTypes.INT.toPython(0L);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the int type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject intType() {
|
||||
return attr("int");
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes sequence types and converts them to lists.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject list(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
PythonObject listF = attr("list");
|
||||
PythonObject ret = listF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Object is not iterable: " + pythonObject.toString());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty list.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject list() {
|
||||
return PythonTypes.LIST.toPython(Collections.emptyList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns list type object.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject listType() {
|
||||
return attr("list");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a dictionary.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject dict(PythonObject pythonObject) {
|
||||
PythonObject dictF = attr("dict");
|
||||
PythonObject ret = dictF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Cannot build dict from object: " + pythonObject.toString());
|
||||
}
|
||||
dictF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty dict
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject dict() {
|
||||
return PythonTypes.DICT.toPython(Collections.emptyMap());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns dict type object.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject dictType() {
|
||||
return attr("dict");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a set.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject set(PythonObject pythonObject) {
|
||||
PythonObject setF = attr("set");
|
||||
PythonObject ret = setF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Cannot build set from object: " + pythonObject.toString());
|
||||
}
|
||||
setF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty set.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject set() {
|
||||
PythonObject setF = attr("set");
|
||||
PythonObject ret;
|
||||
ret = setF.call();
|
||||
setF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty set.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject setType() {
|
||||
return attr("set");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a bytearray.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bytearray(PythonObject pythonObject) {
|
||||
PythonObject baF = attr("bytearray");
|
||||
PythonObject ret = baF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Cannot build bytearray from object: " + pythonObject.toString());
|
||||
}
|
||||
baF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty bytearray.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bytearray() {
|
||||
PythonObject baF = attr("bytearray");
|
||||
PythonObject ret;
|
||||
ret = baF.call();
|
||||
baF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns bytearray type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bytearrayType() {
|
||||
return attr("bytearray");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a memoryview.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject memoryview(PythonObject pythonObject) {
|
||||
PythonObject mvF = attr("memoryview");
|
||||
PythonObject ret = mvF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Cannot build memoryview from object: " + pythonObject.toString());
|
||||
}
|
||||
mvF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns memoryview type object.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject memoryviewType() {
|
||||
return attr("memoryview");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a byte string.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bytes(PythonObject pythonObject) {
|
||||
PythonObject bytesF = attr("bytes");
|
||||
PythonObject ret = bytesF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Cannot build bytes from object: " + pythonObject.toString());
|
||||
}
|
||||
bytesF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty byte string.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bytes() {
|
||||
PythonObject bytesF = attr("bytes");
|
||||
PythonObject ret;
|
||||
ret = bytesF.call();
|
||||
bytesF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns bytes type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject bytesType() {
|
||||
return attr("bytes");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a tuple.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject tuple(PythonObject pythonObject) {
|
||||
PythonObject tupleF = attr("tupleF");
|
||||
PythonObject ret = tupleF.call(pythonObject);
|
||||
if (ret.isNone()) {
|
||||
throw new PythonException("Cannot build tuple from object: " + pythonObject.toString());
|
||||
}
|
||||
tupleF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns empty tuple.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject tuple() {
|
||||
PythonObject tupleF = attr("tuple");
|
||||
PythonObject ret;
|
||||
ret = tupleF.call();
|
||||
tupleF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns tuple type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject tupleType() {
|
||||
return attr("tuple");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an Exception
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject Exception(PythonObject pythonObject) {
|
||||
PythonObject excF = attr("Exception");
|
||||
PythonObject ret = excF.call(pythonObject);
|
||||
excF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an Exception
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject Exception() {
|
||||
PythonObject excF = attr("Exception");
|
||||
PythonObject ret;
|
||||
ret = excF.call();
|
||||
excF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns Exception type object
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject ExceptionType() {
|
||||
return attr("Exception");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the globals dictionary.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject globals() {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject main = PyImport_ImportModule("__main__");
|
||||
PyObject globals = PyModule_GetDict(main);
|
||||
Py_DecRef(main);
|
||||
return new PythonObject(globals, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the type of an object.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject type(PythonObject pythonObject) {
|
||||
PythonObject typeF = attr("type");
|
||||
PythonObject ret = typeF.call(pythonObject);
|
||||
typeF.del();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns True if the specified object is of the specified type, otherwise False.
|
||||
* @param obj
|
||||
* @param type
|
||||
* @return
|
||||
*/
|
||||
public static boolean isinstance(PythonObject obj, PythonObject... type) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject argsTuple = PyTuple_New(type.length);
|
||||
try {
|
||||
for (int i = 0; i < type.length; i++) {
|
||||
PythonObject x = type[i];
|
||||
Py_IncRef(x.getNativePythonObject());
|
||||
PyTuple_SetItem(argsTuple, i, x.getNativePythonObject());
|
||||
}
|
||||
return PyObject_IsInstance(obj.getNativePythonObject(), argsTuple) != 0;
|
||||
} finally {
|
||||
Py_DecRef(argsTuple);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates the specified expression.
|
||||
* @param expression
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject eval(String expression) {
|
||||
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject compiledCode = Py_CompileString(expression, "", Py_eval_input);
|
||||
PyObject main = PyImport_ImportModule("__main__");
|
||||
PyObject globals = PyModule_GetDict(main);
|
||||
PyObject locals = PyDict_New();
|
||||
try {
|
||||
return new PythonObject(PyEval_EvalCode(compiledCode, globals, locals));
|
||||
} finally {
|
||||
Py_DecRef(main);
|
||||
Py_DecRef(locals);
|
||||
Py_DecRef(compiledCode);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the builtins module
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject builtins() {
|
||||
return importModule("builtins");
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns None.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject None() {
|
||||
return eval("None");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns True.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject True() {
|
||||
return eval("True");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns False.
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject False() {
|
||||
return eval("False");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns True if the object passed is callable callable, otherwise False.
|
||||
* @param pythonObject
|
||||
* @return
|
||||
*/
|
||||
public static boolean callable(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
return PyCallable_Check(pythonObject.getNativePythonObject()) == 1;
|
||||
}
|
||||
|
||||
|
||||
public static void setContext(String context){
|
||||
PythonContextManager.setContext(context);
|
||||
}
|
||||
|
||||
public static String getCurrentContext() {
|
||||
return PythonContextManager.getCurrentContext();
|
||||
}
|
||||
|
||||
public static void deleteContext(String context){
|
||||
PythonContextManager.deleteContext(context);
|
||||
}
|
||||
public static void resetContext() {
|
||||
PythonContextManager.reset();
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a string of code.
|
||||
* @param code
|
||||
* @throws PythonException
|
||||
*/
|
||||
public static void exec(String code) throws PythonException {
|
||||
PythonExecutioner.exec(code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a string of code.
|
||||
* @param code
|
||||
* @param inputs
|
||||
* @param outputs
|
||||
*/
|
||||
public static void exec(String code, List<PythonVariable> inputs, List<PythonVariable> outputs){
|
||||
PythonExecutioner.exec(code, inputs, outputs);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.python4j;
|
||||
|
||||
/**
|
||||
*
|
||||
* This class helps control the runtime's {@link PythonExecutioner} -
|
||||
* the {@link PythonExecutioner} is heavily system properties based.
|
||||
* Various aspects of the python executioner can be controlled with
|
||||
* the properties in this class. Python's core behavior of initialization,
|
||||
* python path setting, and working with javacpp's embedded cpython
|
||||
* are keys to integrating the python executioner successfully with various applications.
|
||||
*
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
public class PythonConstants {
|
||||
public final static String DEFAULT_PYTHON_PATH_PROPERTY = "org.eclipse.python4j.path";
|
||||
public final static String JAVACPP_PYTHON_APPEND_TYPE = "org.eclipse.python4j.path.append";
|
||||
//for embedded execution, this is to ensure we allow customization of the gil state releasing when running in another embedded python situation
|
||||
public final static String RELEASE_GIL_AUTOMATICALLY = "org.eclipse.python4j.release_gil_automatically";
|
||||
public final static String DEFAULT_RELEASE_GIL_AUTOMATICALLY = "true";
|
||||
public final static String DEFAULT_APPEND_TYPE = "before";
|
||||
public final static String INITIALIZE_PYTHON = "org.eclipse.python4j.python.initialize";
|
||||
public final static String DEFAULT_INITIALIZE_PYTHON = "true";
|
||||
public final static String PYTHON_EXEC_RESOURCE = "org/nd4j/python4j/pythonexec/pythonexec.py";
|
||||
final static String PYTHON_EXCEPTION_KEY = "__python_exception__";
|
||||
public final static String CREATE_NPY_VIA_PYTHON = "org.eclipse.python4j.create_npy_python";
|
||||
public final static String DEFAULT_CREATE_NPY_VIA_PYTHON = "false";
|
||||
|
||||
/**
|
||||
* Controls how to create the numpy array objects associated
|
||||
* with the NumpyArray.java module.
|
||||
*
|
||||
* Depending on how threading is handled, Py_Type() causes a JVM crash
|
||||
* when used. Py_Type() is used to obtain the type of a numpy array.
|
||||
* Defaults to false, as most of the time this is less performant and not needed.
|
||||
*
|
||||
* The python based method uses raw pointer address + ctypes inline to create the proper numpy array
|
||||
* on the python side.
|
||||
* Otherwise, a more direct c based approach is used.
|
||||
* @return
|
||||
*/
|
||||
public static boolean createNpyViaPython() {
|
||||
return Boolean.parseBoolean(System.getProperty(CREATE_NPY_VIA_PYTHON,DEFAULT_CREATE_NPY_VIA_PYTHON));
|
||||
}
|
||||
|
||||
/**
|
||||
* Setter for the associated property
|
||||
* from {@link #createNpyViaPython()}
|
||||
* please see this function for more information.
|
||||
* @param createNpyViaPython
|
||||
*/
|
||||
public static void setCreateNpyViaPython(boolean createNpyViaPython) {
|
||||
System.setProperty(CREATE_NPY_VIA_PYTHON,String.valueOf(createNpyViaPython));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sets the default python path.
|
||||
* See {@link #defaultPythonPath()}
|
||||
* for more information.
|
||||
* @param newPythonPath the new python path to use
|
||||
*/
|
||||
public static void setDefaultPythonPath(String newPythonPath) {
|
||||
System.setProperty(DEFAULT_PYTHON_PATH_PROPERTY,newPythonPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the default python path.
|
||||
* This python path should be initialized before the {@link PythonExecutioner}
|
||||
* is called.
|
||||
* @return
|
||||
*/
|
||||
public static String defaultPythonPath() {
|
||||
return System.getProperty(PythonConstants.DEFAULT_PYTHON_PATH_PROPERTY);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether to initialize python or not.
|
||||
* This property is used when python should be initialized manually.
|
||||
* Normally, the {@link PythonExecutioner} will handle initialization
|
||||
* in its {@link PythonExecutioner#init()} method
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static boolean initializePython() {
|
||||
return Boolean.parseBoolean(System.getProperty(INITIALIZE_PYTHON,DEFAULT_INITIALIZE_PYTHON));
|
||||
}
|
||||
|
||||
/**
|
||||
* See {@link #initializePython()}
|
||||
* for more information on this property.
|
||||
* This is the setter method for the associated value.
|
||||
* @param initializePython whether to initialize python or not
|
||||
*/
|
||||
public static void setInitializePython(boolean initializePython) {
|
||||
System.setProperty(INITIALIZE_PYTHON,String.valueOf(initializePython));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the default javacpp python append type.
|
||||
* In javacpp's cython module, it comes with built in support
|
||||
* for determining the python path of most modules.
|
||||
*
|
||||
* This can clash when invoking python using another distribution of python
|
||||
* such as anaconda. This property allows the user to control how javacpp
|
||||
* interacts with a different python present on the classpath.
|
||||
*
|
||||
* The default value is {@link #DEFAULT_APPEND_TYPE}
|
||||
* @return
|
||||
*/
|
||||
public static PythonExecutioner.JavaCppPathType javaCppPythonAppendType() {
|
||||
return PythonExecutioner.JavaCppPathType.valueOf(System.getProperty(JAVACPP_PYTHON_APPEND_TYPE,DEFAULT_APPEND_TYPE).toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Setter for the javacpp append type.
|
||||
* See {@link #javaCppPythonAppendType()}
|
||||
* for more information on value set by this setter.
|
||||
* @param appendType the append type to use
|
||||
*/
|
||||
public static void setJavacppPythonAppendType(PythonExecutioner.JavaCppPathType appendType) {
|
||||
System.setProperty(JAVACPP_PYTHON_APPEND_TYPE,appendType.name());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* See {@link #releaseGilAutomatically()}
|
||||
* for more information on this setter.
|
||||
* @param releaseGilAutomatically whether to release the gil automatically or not.
|
||||
*/
|
||||
public static void setReleaseGilAutomatically(boolean releaseGilAutomatically) {
|
||||
System.setProperty(RELEASE_GIL_AUTOMATICALLY,String.valueOf(releaseGilAutomatically));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the GIL is released automatically or not.
|
||||
* For linking against applications where python is already present
|
||||
* this is a knob allowing people to turn automatic python thread management off.
|
||||
* This is enabled by default. See {@link #RELEASE_GIL_AUTOMATICALLY}
|
||||
* and its default value {@link #DEFAULT_RELEASE_GIL_AUTOMATICALLY}
|
||||
* @return
|
||||
*/
|
||||
public final static boolean releaseGilAutomatically() {
|
||||
return Boolean.parseBoolean(System.getProperty(RELEASE_GIL_AUTOMATICALLY,DEFAULT_RELEASE_GIL_AUTOMATICALLY));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
public class PythonContextManager {
|
||||
|
||||
private static Set<String> contexts = new HashSet<>();
|
||||
private static AtomicBoolean init = new AtomicBoolean(false);
|
||||
private static String currentContext;
|
||||
private static final String MAIN_CONTEXT = "main";
|
||||
private static final String COLLAPSED_KEY = "__collapsed__";
|
||||
|
||||
static {
|
||||
init();
|
||||
}
|
||||
|
||||
|
||||
public static class Context implements Closeable{
|
||||
private final String name;
|
||||
private final String previous;
|
||||
private final boolean temp;
|
||||
public Context(){
|
||||
name = "temp_" + UUID.randomUUID().toString().replace("-", "_");
|
||||
temp = true;
|
||||
previous = getCurrentContext();
|
||||
setContext(name);
|
||||
}
|
||||
public Context(String name){
|
||||
this.name = name;
|
||||
temp = false;
|
||||
previous = getCurrentContext();
|
||||
setContext(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close(){
|
||||
setContext(previous);
|
||||
if (temp) deleteContext(name);
|
||||
}
|
||||
}
|
||||
|
||||
private static void init() {
|
||||
if (init.get()) return;
|
||||
new PythonExecutioner();
|
||||
init.set(true);
|
||||
currentContext = MAIN_CONTEXT;
|
||||
contexts.add(currentContext);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Adds a new context.
|
||||
* @param contextName
|
||||
*/
|
||||
public static void addContext(String contextName) {
|
||||
if (!validateContextName(contextName)) {
|
||||
throw new PythonException("Invalid context name: " + contextName);
|
||||
}
|
||||
contexts.add(contextName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if context exists, else false.
|
||||
* @param contextName
|
||||
* @return
|
||||
*/
|
||||
public static boolean hasContext(String contextName) {
|
||||
return contexts.contains(contextName);
|
||||
}
|
||||
|
||||
private static boolean validateContextName(String s) {
|
||||
for (int i=0; i<s.length(); i++){
|
||||
char c = s.toLowerCase().charAt(i);
|
||||
if (i == 0){
|
||||
if (c >= '0' && c <= '9'){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!(c=='_' || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static String getContextPrefix(String contextName) {
|
||||
return COLLAPSED_KEY + contextName + "__";
|
||||
}
|
||||
|
||||
private static String getCollapsedVarNameForContext(String varName, String contextName) {
|
||||
return getContextPrefix(contextName) + varName;
|
||||
}
|
||||
|
||||
private static String expandCollapsedVarName(String varName, String contextName) {
|
||||
String prefix = COLLAPSED_KEY + contextName + "__";
|
||||
return varName.substring(prefix.length());
|
||||
|
||||
}
|
||||
|
||||
private static void collapseContext(String contextName) {
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
PythonObject globals = Python.globals();
|
||||
PythonObject pop = globals.attr("pop");
|
||||
PythonObject keysF = globals.attr("keys");
|
||||
PythonObject keys = keysF.call();
|
||||
PythonObject keysList = Python.list(keys);
|
||||
int numKeys = Python.len(keysList).toInt();
|
||||
for (int i = 0; i < numKeys; i++) {
|
||||
PythonObject key = keysList.get(i);
|
||||
String keyStr = key.toString();
|
||||
if (!((keyStr.startsWith("__") && keyStr.endsWith("__")) || keyStr.startsWith("__collapsed_"))) {
|
||||
String collapsedKey = getCollapsedVarNameForContext(keyStr, contextName);
|
||||
PythonObject val = pop.call(key);
|
||||
|
||||
PythonObject pyNewKey = new PythonObject(collapsedKey);
|
||||
globals.set(pyNewKey, val);
|
||||
}
|
||||
}
|
||||
} catch (Exception pe) {
|
||||
throw new RuntimeException(pe);
|
||||
}
|
||||
}
|
||||
|
||||
private static void expandContext(String contextName) {
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
String prefix = getContextPrefix(contextName);
|
||||
PythonObject globals = Python.globals();
|
||||
PythonObject pop = globals.attr("pop");
|
||||
PythonObject keysF = globals.attr("keys");
|
||||
|
||||
PythonObject keys = keysF.call();
|
||||
|
||||
PythonObject keysList = Python.list(keys);
|
||||
try (PythonGC __ = PythonGC.pause()) {
|
||||
int numKeys = Python.len(keysList).toInt();
|
||||
|
||||
for (int i = 0; i < numKeys; i++) {
|
||||
PythonObject key = keysList.get(i);
|
||||
String keyStr = key.toString();
|
||||
if (keyStr.startsWith(prefix)) {
|
||||
String expandedKey = expandCollapsedVarName(keyStr, contextName);
|
||||
PythonObject val = pop.call(key);
|
||||
PythonObject newKey = new PythonObject(expandedKey);
|
||||
globals.set(newKey, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Activates the specified context
|
||||
* @param contextName
|
||||
*/
|
||||
public static void setContext(String contextName) {
|
||||
if (contextName.equals(currentContext)) {
|
||||
return;
|
||||
}
|
||||
if (!hasContext(contextName)) {
|
||||
addContext(contextName);
|
||||
}
|
||||
|
||||
|
||||
collapseContext(currentContext);
|
||||
|
||||
expandContext(contextName);
|
||||
currentContext = contextName;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Activates the main context
|
||||
*/
|
||||
public static void setMainContext() {
|
||||
setContext(MAIN_CONTEXT);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current context's name.
|
||||
* @return
|
||||
*/
|
||||
public static String getCurrentContext() {
|
||||
return currentContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the current context.
|
||||
*/
|
||||
public static void reset() {
|
||||
String tempContext = "___temp__context___";
|
||||
String currContext = currentContext;
|
||||
setContext(tempContext);
|
||||
deleteContext(currContext);
|
||||
setContext(currContext);
|
||||
deleteContext(tempContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the specified context.
|
||||
* @param contextName
|
||||
*/
|
||||
public static void deleteContext(String contextName) {
|
||||
if (contextName.equals(currentContext)) {
|
||||
throw new PythonException("Cannot delete current context!");
|
||||
}
|
||||
if (!contexts.contains(contextName)) {
|
||||
return;
|
||||
}
|
||||
String prefix = getContextPrefix(contextName);
|
||||
PythonObject globals = Python.globals();
|
||||
PythonObject keysList = Python.list(globals.attr("keys").call());
|
||||
int numKeys = Python.len(keysList).toInt();
|
||||
for (int i = 0; i < numKeys; i++) {
|
||||
PythonObject key = keysList.get(i);
|
||||
String keyStr = key.toString();
|
||||
if (keyStr.startsWith(prefix)) {
|
||||
globals.attr("__delitem__").call(key);
|
||||
}
|
||||
}
|
||||
contexts.remove(contextName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes all contexts except the main context.
|
||||
*/
|
||||
public static void deleteNonMainContexts() {
|
||||
setContext(MAIN_CONTEXT); // will never fail
|
||||
for (String c : contexts.toArray(new String[0])) {
|
||||
if (!c.equals(MAIN_CONTEXT)) {
|
||||
deleteContext(c); // will never fail
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the names of all contexts.
|
||||
* @return
|
||||
*/
|
||||
public String[] getContexts() {
|
||||
return contexts.toArray(new String[0]);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
public class PythonException extends RuntimeException {
|
||||
public PythonException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
private static String getExceptionString(PythonObject exception) {
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
if (Python.isinstance(exception, Python.ExceptionType())) {
|
||||
String exceptionClass = Python.type(exception).attr("__name__").toString();
|
||||
String message = exception.toString();
|
||||
return exceptionClass + ": " + message;
|
||||
}
|
||||
return exception.toString();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("An error occurred while trying to create a PythonException.", e);
|
||||
}
|
||||
}
|
||||
|
||||
public PythonException(PythonObject exception) {
|
||||
this(getExceptionString(exception));
|
||||
}
|
||||
|
||||
public PythonException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public PythonException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.bytedeco.cpython.global.python;
|
||||
import org.nd4j.common.io.ClassPathResource;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
|
||||
/**
|
||||
* PythonExecutioner handles executing python code either from passed in python code
|
||||
* or via a python script.
|
||||
*
|
||||
* PythonExecutioner has a few java system properties to be aware of when executing python:
|
||||
* @link {{@link PythonConstants#DEFAULT_PYTHON_PATH_PROPERTY}} : The default python path to be used by the executioner.
|
||||
* This can be passed with -Dorg.eclipse.python4j.path=your/python/path
|
||||
*
|
||||
* Python4j has a default python path that imports the javacpp python path depending on what is present.
|
||||
* The javacpp python presets such as {@link org.bytedeco.cpython.global.python} have a cachePackages() method
|
||||
* that leverages loading python artifacts from the python path.
|
||||
*
|
||||
* This python path can be merged with a custom one or just used as is.
|
||||
* A user specifies this behavior with the system property {@link PythonConstants#JAVACPP_PYTHON_APPEND_TYPE}
|
||||
* This property can have 3 possible values:
|
||||
* 1. before
|
||||
* 2. after
|
||||
* 3. none
|
||||
*
|
||||
* Order can matter when resolving versions of libraries very similar to the system path. Ensure when adding a custom python path
|
||||
* that these properties are well tested and well understood before use.
|
||||
*
|
||||
* @author Adam Gibson, Fariz Rahman
|
||||
*/
|
||||
public class PythonExecutioner {
|
||||
private static AtomicBoolean init = new AtomicBoolean(false);
|
||||
|
||||
static {
|
||||
init();
|
||||
}
|
||||
|
||||
public static synchronized void init() {
|
||||
if (init.get()) {
|
||||
return;
|
||||
}
|
||||
|
||||
init.set(true);
|
||||
initPythonPath();
|
||||
if(PythonConstants.initializePython())
|
||||
Py_InitializeEx(0);
|
||||
//initialize separately to ensure that numpy import array is not imported twice
|
||||
for (PythonType type: PythonTypes.get()) {
|
||||
type.init();
|
||||
}
|
||||
|
||||
//set the main thread state for the gil
|
||||
PythonGIL.setMainThreadState();
|
||||
if(Py_IsFinalizing() != 1 && PythonConstants.releaseGilAutomatically())
|
||||
PyEval_SaveThread();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a variable.
|
||||
*
|
||||
* @param name
|
||||
* @param value
|
||||
*/
|
||||
public static void setVariable(String name, PythonObject value) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject main = PyImport_ImportModule("__main__");
|
||||
PyObject globals = PyModule_GetDict(main);
|
||||
PyDict_SetItemString(globals, name, value.getNativePythonObject());
|
||||
Py_DecRef(main);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets given list of PythonVariables in the interpreter.
|
||||
*
|
||||
* @param pyVars
|
||||
*/
|
||||
public static void setVariables(List<PythonVariable> pyVars) {
|
||||
for (PythonVariable pyVar : pyVars)
|
||||
setVariable(pyVar.getName(), pyVar.getPythonObject());
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets given list of PythonVariables in the interpreter.
|
||||
*
|
||||
* @param pyVars
|
||||
*/
|
||||
public static void setVariables(PythonVariable... pyVars) {
|
||||
setVariables(Arrays.asList(pyVars));
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the given list of PythonVariables from the interpreter.
|
||||
*
|
||||
* @param pyVars
|
||||
*/
|
||||
public static void getVariables(List<PythonVariable> pyVars) {
|
||||
for (PythonVariable pyVar : pyVars)
|
||||
pyVar.setValue(getVariable(pyVar.getName(), pyVar.getType()).getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the given list of PythonVariables from the interpreter.
|
||||
*
|
||||
* @param pyVars
|
||||
*/
|
||||
public static void getVariables(PythonVariable... pyVars) {
|
||||
getVariables(Arrays.asList(pyVars));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Gets the variable with the given name from the interpreter.
|
||||
*
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
public static PythonObject getVariable(String name) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject main = PyImport_ImportModule("__main__");
|
||||
PyObject globals = PyModule_GetDict(main);
|
||||
PyObject pyName = PyUnicode_FromString(name);
|
||||
try {
|
||||
if (PyDict_Contains(globals, pyName) == 1) {
|
||||
return new PythonObject(PyObject_GetItem(globals, pyName), false);
|
||||
}
|
||||
} finally {
|
||||
Py_DecRef(main);
|
||||
//Py_DecRef(globals);
|
||||
Py_DecRef(pyName);
|
||||
}
|
||||
return new PythonObject(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the variable with the given name from the interpreter.
|
||||
*
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
public static <T> PythonVariable<T> getVariable(String name, PythonType<T> type) {
|
||||
PythonObject val = getVariable(name);
|
||||
return new PythonVariable<>(name, type, type.toJava(val));
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a string of code
|
||||
*
|
||||
* @param code
|
||||
*/
|
||||
public static synchronized void simpleExec(String code) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
|
||||
int result = PyRun_SimpleStringFlags(code, null);
|
||||
if (result != 0) {
|
||||
throw new PythonException("Execution failed, unable to retrieve python exception.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void throwIfExecutionFailed() {
|
||||
PythonObject ex = getVariable(PythonConstants.PYTHON_EXCEPTION_KEY);
|
||||
if (ex != null && !ex.isNone() && !ex.toString().isEmpty()) {
|
||||
setVariable(PythonConstants.PYTHON_EXCEPTION_KEY, PythonTypes.STR.toPython(""));
|
||||
throw new PythonException(ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String getWrappedCode(String code) {
|
||||
ClassPathResource resource = new ClassPathResource(PythonConstants.PYTHON_EXEC_RESOURCE);
|
||||
if(!resource.exists()) {
|
||||
throw new IllegalStateException("Unable to find class path resource for python script execution: " + PythonConstants.PYTHON_EXEC_RESOURCE + " if using via graalvm, please ensure this resource is included in your resources-config.json");
|
||||
}
|
||||
try (InputStream is = resource.getInputStream()) {
|
||||
String base = IOUtils.toString(is, StandardCharsets.UTF_8);
|
||||
String indentedCode = " " + code.replace("\n", "\n ");
|
||||
String out = base.replace(" pass", indentedCode);
|
||||
return out;
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("Unable to read python code!", e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a string of code. Throws PythonException if execution fails.
|
||||
*
|
||||
* @param code
|
||||
*/
|
||||
public static void exec(String code) {
|
||||
simpleExec(getWrappedCode(code));
|
||||
throwIfExecutionFailed();
|
||||
}
|
||||
|
||||
public static void exec(String code, List<PythonVariable> inputs, List<PythonVariable> outputs) {
|
||||
if (inputs != null) {
|
||||
setVariables(inputs.toArray(new PythonVariable[0]));
|
||||
}
|
||||
exec(code);
|
||||
if (outputs != null) {
|
||||
getVariables(outputs.toArray(new PythonVariable[0]));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return list of all supported variables in the interpreter.
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static PythonVariables getAllVariables() {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PythonVariables ret = new PythonVariables();
|
||||
PyObject main = PyImport_ImportModule("__main__");
|
||||
PyObject globals = PyModule_GetDict(main);
|
||||
PyObject keys = PyDict_Keys(globals);
|
||||
PyObject keysIter = PyObject_GetIter(keys);
|
||||
try {
|
||||
|
||||
long n = PyObject_Size(globals);
|
||||
for (int i = 0; i < n; i++) {
|
||||
PyObject pyKey = PyIter_Next(keysIter);
|
||||
try {
|
||||
if (!new PythonObject(pyKey, false).toString().startsWith("_")) {
|
||||
|
||||
PyObject pyVal = PyObject_GetItem(globals, pyKey); // TODO check ref count
|
||||
PythonType pt;
|
||||
try {
|
||||
pt = PythonTypes.getPythonTypeForPythonObject(new PythonObject(pyVal, false));
|
||||
|
||||
} catch (PythonException pe) {
|
||||
pt = null;
|
||||
}
|
||||
if (pt != null) {
|
||||
ret.add(
|
||||
new PythonVariable<>(
|
||||
new PythonObject(pyKey, false).toString(),
|
||||
pt,
|
||||
pt.toJava(new PythonObject(pyVal, false))
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
Py_DecRef(pyKey);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
Py_DecRef(keysIter);
|
||||
Py_DecRef(keys);
|
||||
Py_DecRef(main);
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Executes a string of code and returns a list of all supported variables.
|
||||
*
|
||||
* @param code
|
||||
* @param inputs
|
||||
* @return
|
||||
*/
|
||||
public static PythonVariables execAndReturnAllVariables(String code, List<PythonVariable> inputs) {
|
||||
setVariables(inputs);
|
||||
simpleExec(getWrappedCode(code));
|
||||
return getAllVariables();
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a string of code and returns a list of all supported variables.
|
||||
*
|
||||
* @param code
|
||||
* @return
|
||||
*/
|
||||
public static PythonVariables execAndReturnAllVariables(String code) {
|
||||
simpleExec(getWrappedCode(code));
|
||||
return getAllVariables();
|
||||
}
|
||||
|
||||
private static synchronized void initPythonPath() {
|
||||
try {
|
||||
String path = PythonConstants.defaultPythonPath();
|
||||
|
||||
List<File> packagesList = new ArrayList<>();
|
||||
packagesList.addAll(Arrays.asList(cachePackages()));
|
||||
for (PythonType type: PythonTypes.get()){
|
||||
packagesList.addAll(Arrays.asList(type.packages()));
|
||||
}
|
||||
//// TODO: fix in javacpp
|
||||
packagesList.add(new File(python.cachePackage(), "site-packages"));
|
||||
|
||||
File[] packages = packagesList.toArray(new File[0]);
|
||||
|
||||
if (path == null) {
|
||||
for (File packageDir : packages) {
|
||||
PyObject pythonPath = PyUnicode_FromString(packageDir.getAbsolutePath());
|
||||
PySys_SetObject("path", pythonPath);
|
||||
Py_DecRef(pythonPath);
|
||||
}
|
||||
} else {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
|
||||
JavaCppPathType pathAppendValue = PythonConstants.javaCppPythonAppendType();
|
||||
switch (pathAppendValue) {
|
||||
case BEFORE:
|
||||
for (File cacheDir : packages) {
|
||||
sb.append(cacheDir);
|
||||
sb.append(java.io.File.pathSeparator);
|
||||
}
|
||||
|
||||
sb.append(path);
|
||||
break;
|
||||
case AFTER:
|
||||
sb.append(path);
|
||||
|
||||
for (File cacheDir : packages) {
|
||||
sb.append(cacheDir);
|
||||
sb.append(java.io.File.pathSeparator);
|
||||
}
|
||||
break;
|
||||
case NONE:
|
||||
sb.append(path);
|
||||
break;
|
||||
}
|
||||
|
||||
PyObject pythonPath = PyUnicode_FromString(sb.toString());
|
||||
PySys_SetObject("path", pythonPath);
|
||||
Py_DecRef(pythonPath);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public enum JavaCppPathType {
|
||||
BEFORE, AFTER, NONE
|
||||
}
|
||||
|
||||
private static File[] cachePackages() throws IOException {
|
||||
File[] path = org.bytedeco.cpython.global.python.cachePackages();
|
||||
path = Arrays.copyOf(path, path.length + 1);
|
||||
path[path.length - 1] = cachePackage();
|
||||
return path;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
|
||||
public class PythonGC implements Closeable {
|
||||
|
||||
private PythonGC previousFrame = null;
|
||||
private boolean active = true;
|
||||
private static PythonGC currentFrame = new PythonGC();
|
||||
|
||||
private Set<PyObject> objects = new HashSet<>();
|
||||
|
||||
private boolean alreadyRegistered(PyObject pyObject) {
|
||||
if (objects.contains(pyObject)) {
|
||||
return true;
|
||||
}
|
||||
if (previousFrame == null) {
|
||||
return false;
|
||||
}
|
||||
return previousFrame.alreadyRegistered(pyObject);
|
||||
|
||||
}
|
||||
|
||||
private void addObject(PythonObject pythonObject) {
|
||||
if (!active) return;
|
||||
if (Pointer.isNull(pythonObject.getNativePythonObject()))return;
|
||||
if (alreadyRegistered(pythonObject.getNativePythonObject())) {
|
||||
return;
|
||||
}
|
||||
objects.add(pythonObject.getNativePythonObject());
|
||||
}
|
||||
|
||||
public static void register(PythonObject pythonObject) {
|
||||
currentFrame.addObject(pythonObject);
|
||||
}
|
||||
|
||||
public static void keep(PythonObject pythonObject) {
|
||||
currentFrame.objects.remove(pythonObject.getNativePythonObject());
|
||||
if (currentFrame.previousFrame != null) {
|
||||
currentFrame.previousFrame.addObject(pythonObject);
|
||||
}
|
||||
}
|
||||
|
||||
private PythonGC() {
|
||||
}
|
||||
|
||||
public static PythonGC watch() {
|
||||
PythonGC ret = new PythonGC();
|
||||
ret.previousFrame = currentFrame;
|
||||
ret.active = currentFrame.active;
|
||||
currentFrame = ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void collect() {
|
||||
for (PyObject pyObject : objects) {
|
||||
// TODO find out how globals gets collected here
|
||||
if (pyObject.equals(Python.globals().getNativePythonObject())) continue;
|
||||
// try{
|
||||
// System.out.println(PythonTypes.STR.toJava(new PythonObject(pyObject, false)));
|
||||
// }catch (Exception e){}
|
||||
Py_DecRef(pyObject);
|
||||
|
||||
}
|
||||
this.objects = new HashSet<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (active) collect();
|
||||
currentFrame = previousFrame;
|
||||
}
|
||||
|
||||
public static boolean isWatching() {
|
||||
if (!currentFrame.active) return false;
|
||||
return currentFrame.previousFrame != null;
|
||||
}
|
||||
|
||||
public static PythonGC pause() {
|
||||
PythonGC pausedFrame = new PythonGC();
|
||||
pausedFrame.active = false;
|
||||
pausedFrame.previousFrame = currentFrame;
|
||||
currentFrame = pausedFrame;
|
||||
return pausedFrame;
|
||||
}
|
||||
|
||||
public static void resume() {
|
||||
if (currentFrame.active) {
|
||||
throw new RuntimeException("GC not paused!");
|
||||
}
|
||||
currentFrame = currentFrame.previousFrame;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.bytedeco.cpython.PyThreadState;
|
||||
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
|
||||
@Slf4j
|
||||
public class PythonGIL implements AutoCloseable {
|
||||
private static final AtomicBoolean acquired = new AtomicBoolean();
|
||||
private boolean acquiredByMe = false;
|
||||
private static long defaultThreadId = -1;
|
||||
private int gilState;
|
||||
private static PyThreadState mainThreadState;
|
||||
private static long mainThreadId = -1;
|
||||
static {
|
||||
new PythonExecutioner();
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the main thread state
|
||||
* based on the current thread calling this method.
|
||||
* This method should not be called by the user.
|
||||
* It is already invoked automatically in {@link PythonExecutioner}
|
||||
*/
|
||||
public static synchronized void setMainThreadState() {
|
||||
if(mainThreadId < 0 && mainThreadState != null) {
|
||||
mainThreadState = PyThreadState_Get();
|
||||
mainThreadId = Thread.currentThread().getId();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that the lock has been acquired.
|
||||
*/
|
||||
public static void assertThreadSafe() {
|
||||
if (acquired.get()) {
|
||||
return;
|
||||
}
|
||||
if (defaultThreadId == -1) {
|
||||
defaultThreadId = Thread.currentThread().getId();
|
||||
} else if (defaultThreadId != Thread.currentThread().getId()) {
|
||||
throw new RuntimeException("Attempt to use Python4j from multiple threads without " +
|
||||
"acquiring GIL. Enclose your code in a try(PythonGIL gil = PythonGIL.lock()){...}" +
|
||||
" block to ensure that GIL is acquired in multi-threaded environments.");
|
||||
}
|
||||
|
||||
if(!acquired.get()) {
|
||||
throw new IllegalStateException("Execution happening outside of GIL. Please use PythonExecutioner within a GIL block by wrapping it in a call via: try(PythonGIL gil = PythonGIL.lock()) { .. }");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private PythonGIL() {
|
||||
log.debug("Acquiring GIL on " + Thread.currentThread().getId());
|
||||
acquired.set(true);
|
||||
acquiredByMe = true;
|
||||
acquire();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
if (acquiredByMe) {
|
||||
release();
|
||||
log.info("Releasing GIL on thread " + Thread.currentThread().getId());
|
||||
acquired.set(false);
|
||||
acquiredByMe = false;
|
||||
}
|
||||
else {
|
||||
log.info("Attempted to release GIL without having acquired GIL on thread " + Thread.currentThread().getId());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Lock the GIL for running python scripts.
|
||||
* This method should be used to create a new
|
||||
* {@link PythonGIL} object in the form of:
|
||||
* try(PythonGIL gil = PythonGIL.lock()) {
|
||||
* //your python code here
|
||||
* }
|
||||
* @return the gil for this instance
|
||||
*/
|
||||
public static synchronized PythonGIL lock() {
|
||||
return new PythonGIL();
|
||||
}
|
||||
|
||||
private synchronized void acquire() {
|
||||
if(Thread.currentThread().getId() != mainThreadId) {
|
||||
log.info("Pre Gil State ensure for thread " + Thread.currentThread().getId());
|
||||
gilState = PyGILState_Ensure();
|
||||
log.info("Thread " + Thread.currentThread().getId() + " acquired GIL");
|
||||
} else {
|
||||
//See: https://github.com/pytorch/pytorch/issues/47776#issuecomment-726455206
|
||||
//From this thread: // PyEval_RestoreThread() should not be called if runtime is finalizing
|
||||
// See https://docs.python.org/3/c-api/init.html#c.PyEval_RestoreThread
|
||||
|
||||
if(Py_IsFinalizing() != 1 && PythonConstants.releaseGilAutomatically())
|
||||
PyEval_RestoreThread(mainThreadState);
|
||||
}
|
||||
}
|
||||
|
||||
private void release() { // do not synchronize!
|
||||
if(Thread.currentThread().getId() != mainThreadId) {
|
||||
log.debug("Pre gil state release for thread " + Thread.currentThread().getId());
|
||||
if(PythonConstants.releaseGilAutomatically())
|
||||
PyGILState_Release(gilState);
|
||||
}
|
||||
else {
|
||||
//See: https://github.com/pytorch/pytorch/issues/47776#issuecomment-726455206
|
||||
//From this thread: // PyEval_RestoreThread() should not be called if runtime is finalizing
|
||||
// See https://docs.python.org/3/c-api/init.html#c.PyEval_RestoreThread
|
||||
|
||||
if(Py_IsFinalizing() != 1 && PythonConstants.releaseGilAutomatically())
|
||||
PyEval_RestoreThread(mainThreadState);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the GIL is currently in use.
|
||||
* This is typically true when {@link #lock()}
|
||||
* @return
|
||||
*/
|
||||
public static boolean locked() {
|
||||
return acquired.get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
|
||||
/**
|
||||
* Base interface for a python interpreter for running python code.
|
||||
* Based on:
|
||||
* https://github.com/invesdwin/invesdwin-context-python/blob/master/invesdwin-context-python-parent/invesdwin-context-python-runtime-python4j/src/main/java/de/invesdwin/context/python/runtime/python4j/internal/IPythonEngineWrapper.java
|
||||
*
|
||||
* Permission given here: https://github.com/eclipse/deeplearning4j/issues/9595
|
||||
*
|
||||
* @author Adam Gibson
|
||||
*/
|
||||
public interface PythonInterpreter {
|
||||
|
||||
|
||||
Object getCachedPython(String varName);
|
||||
|
||||
Object getCachedJava(String varName);
|
||||
|
||||
Pair<PythonObject,Object> getCachedPythonJava(String varName);
|
||||
|
||||
/**
|
||||
* GIL Lock object
|
||||
* @return
|
||||
*/
|
||||
GILLock gilLock();
|
||||
|
||||
/**
|
||||
* Execute the given python code
|
||||
* @param expression a python expression
|
||||
*/
|
||||
void exec(String expression);
|
||||
|
||||
/**
|
||||
* Retrieve a variable from the interpreter.
|
||||
* Returns none if none is found or variable
|
||||
* does not exist
|
||||
*
|
||||
* @param variable the variable to retrieve
|
||||
* @param getNew
|
||||
* @return null if the object is none or the variable does not exist
|
||||
*/
|
||||
Object get(String variable, boolean getNew);
|
||||
|
||||
/**
|
||||
* Set a variable in the interpreter
|
||||
* @param variable the variable to set
|
||||
* @param value the value to set
|
||||
*/
|
||||
void set(String variable, Object value);
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
|
||||
public class PythonObject {
|
||||
|
||||
static {
|
||||
new PythonExecutioner();
|
||||
}
|
||||
|
||||
private boolean owned = true;
|
||||
private PyObject nativePythonObject;
|
||||
|
||||
|
||||
public PythonObject(PyObject nativePythonObject, boolean owned) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
this.nativePythonObject = nativePythonObject;
|
||||
this.owned = owned;
|
||||
if (owned && nativePythonObject != null) {
|
||||
PythonGC.register(this);
|
||||
}
|
||||
}
|
||||
|
||||
public PythonObject(PyObject nativePythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
this.nativePythonObject = nativePythonObject;
|
||||
if (nativePythonObject != null) {
|
||||
PythonGC.register(this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public PyObject getNativePythonObject() {
|
||||
return nativePythonObject;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return PythonTypes.STR.toJava(this);
|
||||
|
||||
}
|
||||
|
||||
public boolean isNone() {
|
||||
if (nativePythonObject == null || Pointer.isNull(nativePythonObject)) {
|
||||
return true;
|
||||
}
|
||||
try (PythonGC gc = PythonGC.pause()) {
|
||||
PythonObject type = Python.type(this);
|
||||
boolean ret = Python.type(this).toString().equals("<class 'NoneType'>") && toString().equals("None");
|
||||
Py_DecRef(type.nativePythonObject);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
public void del() {
|
||||
PythonGIL.assertThreadSafe();
|
||||
if (owned && nativePythonObject != null && !PythonGC.isWatching()) {
|
||||
Py_DecRef(nativePythonObject);
|
||||
nativePythonObject = null;
|
||||
}
|
||||
}
|
||||
|
||||
public PythonObject callWithArgs(PythonObject args) {
|
||||
return callWithArgsAndKwargs(args, null);
|
||||
}
|
||||
|
||||
public PythonObject callWithKwargs(PythonObject kwargs) {
|
||||
if (!Python.callable(this)) {
|
||||
throw new PythonException("Object is not callable: " + toString());
|
||||
}
|
||||
PyObject tuple = PyTuple_New(0);
|
||||
PyObject dict = kwargs.nativePythonObject;
|
||||
if (PyObject_IsInstance(dict, new PyObject(PyDict_Type())) != 1) {
|
||||
throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString());
|
||||
}
|
||||
PythonObject ret = new PythonObject(PyObject_Call(nativePythonObject, tuple, dict));
|
||||
Py_DecRef(tuple);
|
||||
return ret;
|
||||
}
|
||||
|
||||
public PythonObject callWithArgsAndKwargs(PythonObject args, PythonObject kwargs) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject tuple = null;
|
||||
boolean ownsTuple = false;
|
||||
try {
|
||||
if (!Python.callable(this)) {
|
||||
throw new PythonException("Object is not callable: " + toString());
|
||||
}
|
||||
|
||||
if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyTuple_Type())) == 1) {
|
||||
tuple = args.nativePythonObject;
|
||||
} else if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyList_Type())) == 1) {
|
||||
tuple = PyList_AsTuple(args.nativePythonObject);
|
||||
ownsTuple = true;
|
||||
} else {
|
||||
throw new PythonException("Expected args to be tuple or list. Received: " + args.toString());
|
||||
}
|
||||
if (kwargs != null && PyObject_IsInstance(kwargs.nativePythonObject, new PyObject(PyDict_Type())) != 1) {
|
||||
throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString());
|
||||
}
|
||||
return new PythonObject(PyObject_Call(nativePythonObject, tuple, kwargs == null ? null : kwargs.nativePythonObject));
|
||||
} finally {
|
||||
if (ownsTuple) Py_DecRef(tuple);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public PythonObject call(Object... args) {
|
||||
return callWithArgsAndKwargs(Arrays.asList(args), null);
|
||||
}
|
||||
|
||||
public PythonObject callWithArgs(List args) {
|
||||
return call(args, null);
|
||||
}
|
||||
|
||||
public PythonObject callWithKwargs(Map kwargs) {
|
||||
return call(null, kwargs);
|
||||
}
|
||||
|
||||
public PythonObject callWithArgsAndKwargs(List args, Map kwargs) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
if (!Python.callable(this)) {
|
||||
throw new PythonException("Object is not callable: " + toString());
|
||||
}
|
||||
PythonObject pyArgs;
|
||||
PythonObject pyKwargs;
|
||||
|
||||
if (args == null || args.isEmpty()) {
|
||||
pyArgs = new PythonObject(PyTuple_New(0));
|
||||
} else {
|
||||
PythonObject argsList = PythonTypes.convert(args);
|
||||
pyArgs = new PythonObject(PyList_AsTuple(argsList.getNativePythonObject()));
|
||||
}
|
||||
if (kwargs == null) {
|
||||
pyKwargs = null;
|
||||
} else {
|
||||
pyKwargs = PythonTypes.convert(kwargs);
|
||||
}
|
||||
|
||||
PythonObject ret = new PythonObject(
|
||||
PyObject_Call(
|
||||
nativePythonObject,
|
||||
pyArgs.nativePythonObject,
|
||||
pyKwargs == null ? null : pyKwargs.nativePythonObject
|
||||
)
|
||||
);
|
||||
|
||||
PythonGC.keep(ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public PythonObject attr(String attrName) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
return new PythonObject(PyObject_GetAttrString(nativePythonObject, attrName));
|
||||
}
|
||||
|
||||
|
||||
public PythonObject(Object javaObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
if (javaObject instanceof PythonObject) {
|
||||
owned = false;
|
||||
nativePythonObject = ((PythonObject) javaObject).nativePythonObject;
|
||||
} else {
|
||||
try (PythonGC gc = PythonGC.pause()) {
|
||||
nativePythonObject = PythonTypes.convert(javaObject).getNativePythonObject();
|
||||
}
|
||||
PythonGC.register(this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public int toInt() {
|
||||
return PythonTypes.INT.toJava(this).intValue();
|
||||
}
|
||||
|
||||
public long toLong() {
|
||||
return PythonTypes.INT.toJava(this);
|
||||
}
|
||||
|
||||
public float toFloat() {
|
||||
return PythonTypes.FLOAT.toJava(this).floatValue();
|
||||
}
|
||||
|
||||
public double toDouble() {
|
||||
return PythonTypes.FLOAT.toJava(this);
|
||||
}
|
||||
|
||||
public boolean toBoolean() {
|
||||
return PythonTypes.BOOL.toJava(this);
|
||||
|
||||
}
|
||||
|
||||
public List toList() {
|
||||
return PythonTypes.LIST.toJava(this);
|
||||
}
|
||||
|
||||
public Map toMap() {
|
||||
return PythonTypes.DICT.toJava(this);
|
||||
}
|
||||
|
||||
public PythonObject get(int key) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
return new PythonObject(PyObject_GetItem(nativePythonObject, PyLong_FromLong(key)));
|
||||
}
|
||||
|
||||
public PythonObject get(String key) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
return new PythonObject(PyObject_GetItem(nativePythonObject, PyUnicode_FromString(key)));
|
||||
}
|
||||
|
||||
public PythonObject get(PythonObject key) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
return new PythonObject(PyObject_GetItem(nativePythonObject, key.nativePythonObject));
|
||||
}
|
||||
|
||||
public void set(PythonObject key, PythonObject value) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject_SetItem(nativePythonObject, key.nativePythonObject, value.nativePythonObject);
|
||||
}
|
||||
|
||||
|
||||
public PythonObject abs(){
|
||||
return new PythonObject(PyNumber_Absolute(nativePythonObject));
|
||||
}
|
||||
public PythonObject add(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_Add(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
public PythonObject sub(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_Subtract(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
public PythonObject mod(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_Divmod(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
public PythonObject mul(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_Multiply(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
public PythonObject trueDiv(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_TrueDivide(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
public PythonObject floorDiv(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_FloorDivide(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
public PythonObject matMul(PythonObject pythonObject){
|
||||
return new PythonObject(PyNumber_MatrixMultiply(nativePythonObject, pythonObject.nativePythonObject));
|
||||
}
|
||||
|
||||
public void addi(PythonObject pythonObject){
|
||||
PyNumber_InPlaceAdd(nativePythonObject, pythonObject.nativePythonObject);
|
||||
}
|
||||
public void subi(PythonObject pythonObject){
|
||||
PyNumber_InPlaceSubtract(nativePythonObject, pythonObject.nativePythonObject);
|
||||
}
|
||||
public void muli(PythonObject pythonObject){
|
||||
PyNumber_InPlaceMultiply(nativePythonObject, pythonObject.nativePythonObject);
|
||||
}
|
||||
public void trueDivi(PythonObject pythonObject){
|
||||
PyNumber_InPlaceTrueDivide(nativePythonObject, pythonObject.nativePythonObject);
|
||||
}
|
||||
public void floorDivi(PythonObject pythonObject){
|
||||
PyNumber_InPlaceFloorDivide(nativePythonObject, pythonObject.nativePythonObject);
|
||||
}
|
||||
public void matMuli(PythonObject pythonObject){
|
||||
PyNumber_InPlaceMatrixMultiply(nativePythonObject, pythonObject.nativePythonObject);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.bytedeco.javacpp.Loader;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class PythonProcess {
|
||||
private static String pythonExecutable = Loader.load(org.bytedeco.cpython.python.class);
|
||||
public static String runAndReturn(String... arguments)throws IOException, InterruptedException{
|
||||
String[] allArgs = new String[arguments.length + 1];
|
||||
for (int i = 0; i < arguments.length; i++){
|
||||
allArgs[i + 1] = arguments[i];
|
||||
}
|
||||
allArgs[0] = pythonExecutable;
|
||||
ProcessBuilder pb = new ProcessBuilder(allArgs);
|
||||
Process process = pb.start();
|
||||
String out = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8);
|
||||
process.waitFor();
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
public static void run(String... arguments)throws IOException, InterruptedException{
|
||||
String[] allArgs = new String[arguments.length + 1];
|
||||
for (int i = 0; i < arguments.length; i++){
|
||||
allArgs[i + 1] = arguments[i];
|
||||
}
|
||||
allArgs[0] = pythonExecutable;
|
||||
ProcessBuilder pb = new ProcessBuilder(allArgs);
|
||||
pb.inheritIO().start().waitFor();
|
||||
}
|
||||
public static void pipInstall(String packageName) throws PythonException{
|
||||
try{
|
||||
run("-m", "pip", "install", packageName);
|
||||
}catch(Exception e){
|
||||
throw new PythonException("Error installing package " + packageName, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void pipInstall(String packageName, String version){
|
||||
pipInstall(packageName + "==" + version);
|
||||
}
|
||||
|
||||
public static void pipUninstall(String packageName) throws PythonException{
|
||||
try{
|
||||
run("-m", "pip", "uninstall", packageName);
|
||||
}catch(Exception e){
|
||||
throw new PythonException("Error uninstalling package " + packageName, e);
|
||||
}
|
||||
|
||||
}
|
||||
public static void pipInstallFromGit(String gitRepoUrl){
|
||||
if (!gitRepoUrl.contains("://")){
|
||||
gitRepoUrl = "git://" + gitRepoUrl;
|
||||
}
|
||||
try{
|
||||
run("-m", "pip", "install", "git+", gitRepoUrl);
|
||||
}catch(Exception e){
|
||||
throw new PythonException("Error installing package from " + gitRepoUrl, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static String getPackageVersion(String packageName){
|
||||
String out;
|
||||
try{
|
||||
out = runAndReturn("-m", "pip", "show", packageName);
|
||||
} catch (Exception e){
|
||||
throw new PythonException("Error finding version for package " + packageName, e);
|
||||
}
|
||||
|
||||
if (!out.contains("Version: ")){
|
||||
throw new PythonException("Can't find package " + packageName);
|
||||
}
|
||||
String pkgVersion = out.split("Version: ")[1].split(System.lineSeparator())[0];
|
||||
return pkgVersion;
|
||||
}
|
||||
|
||||
public static boolean isPackageInstalled(String packageName){
|
||||
try{
|
||||
String out = runAndReturn("-m", "pip", "show", packageName);
|
||||
return !out.isEmpty();
|
||||
}catch (Exception e){
|
||||
throw new PythonException("Error checking if package is installed: " +packageName, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void pipInstallFromRequirementsTxt(String path){
|
||||
try{
|
||||
run("-m", "pip", "install","-r", path);
|
||||
}catch (Exception e){
|
||||
throw new PythonException("Error installing packages from " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
public static void pipInstallFromSetupScript(String path, boolean inplace){
|
||||
|
||||
try{
|
||||
run(path, inplace?"develop":"install");
|
||||
}catch (Exception e){
|
||||
throw new PythonException("Error installing package from " + path, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public abstract class PythonType<T> {
|
||||
|
||||
private final String name;
|
||||
private final Class<T> javaType;
|
||||
|
||||
public PythonType(String name, Class<T> javaType) {
|
||||
this.name = name;
|
||||
this.javaType = javaType;
|
||||
}
|
||||
|
||||
public T adapt(Object javaObject) throws PythonException {
|
||||
return (T) javaObject;
|
||||
}
|
||||
|
||||
public abstract T toJava(PythonObject pythonObject);
|
||||
|
||||
public abstract PythonObject toPython(T javaObject);
|
||||
|
||||
public boolean accepts(Object javaObject) {
|
||||
return javaType.isAssignableFrom(javaObject.getClass());
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj){
|
||||
if (!(obj instanceof PythonType)){
|
||||
return false;
|
||||
}
|
||||
PythonType other = (PythonType)obj;
|
||||
return this.getClass().equals(other.getClass()) && this.name.equals(other.name);
|
||||
}
|
||||
|
||||
public PythonObject pythonType(){
|
||||
return null;
|
||||
}
|
||||
|
||||
public File[] packages(){
|
||||
return new File[0];
|
||||
}
|
||||
|
||||
public void init(){ //not to be called from constructor
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
|
||||
public class PythonTypes {
|
||||
|
||||
|
||||
private static List<PythonType> getPrimitiveTypes() {
|
||||
return Arrays.asList(STR, INT, FLOAT, BOOL, BYTES,new NoneType());
|
||||
}
|
||||
|
||||
private static List<PythonType> getCollectionTypes() {
|
||||
return Arrays.asList(LIST, DICT);
|
||||
}
|
||||
|
||||
|
||||
private static List<PythonType> getExternalTypes() {
|
||||
List<PythonType> ret = new ArrayList<>();
|
||||
ServiceLoader<PythonType> sl = ServiceLoader.load(PythonType.class);
|
||||
Iterator<PythonType> iter = sl.iterator();
|
||||
while (iter.hasNext()) {
|
||||
ret.add(iter.next());
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static List<PythonType> get() {
|
||||
List<PythonType> ret = new ArrayList<>();
|
||||
ret.addAll(getPrimitiveTypes());
|
||||
ret.addAll(getCollectionTypes());
|
||||
ret.addAll(getExternalTypes());
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static <T> PythonType<T> get(String name) {
|
||||
for (PythonType pt : get()) {
|
||||
if (pt.getName().equals(name)) { // TODO use map instead?
|
||||
return pt;
|
||||
}
|
||||
|
||||
}
|
||||
throw new PythonException("Unknown python type: " + name);
|
||||
}
|
||||
|
||||
|
||||
public static PythonType getPythonTypeForJavaObject(Object javaObject) {
|
||||
for (PythonType pt : get()) {
|
||||
if (pt.accepts(javaObject)) {
|
||||
return pt;
|
||||
}
|
||||
}
|
||||
throw new PythonException("Unable to find python type for java type: " + javaObject.getClass());
|
||||
}
|
||||
|
||||
public static <T> PythonType<T> getPythonTypeForPythonObject(PythonObject pythonObject) {
|
||||
PyObject pyType = PyObject_Type(pythonObject.getNativePythonObject());
|
||||
try {
|
||||
String pyTypeStr = PythonTypes.STR.toJava(new PythonObject(pyType, false));
|
||||
|
||||
for (PythonType pt : get()) {
|
||||
String pyTypeStr2 = "<class '" + pt.getName() + "'>";
|
||||
if (pyTypeStr.equals(pyTypeStr2)) {
|
||||
return pt;
|
||||
} else {
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
PythonObject pyType2 = pt.pythonType();
|
||||
if (pyType2 != null && Python.isinstance(pythonObject, pyType2)) {
|
||||
return pt;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
throw new PythonException("Unable to find converter for python object of type " + pyTypeStr);
|
||||
} finally {
|
||||
Py_DecRef(pyType);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
public static PythonObject convert(Object javaObject) {
|
||||
PythonType pt = getPythonTypeForJavaObject(javaObject);
|
||||
return pt.toPython(pt.adapt(javaObject));
|
||||
}
|
||||
|
||||
public static final PythonType<String> STR = new PythonType<String>("str", String.class) {
|
||||
|
||||
@Override
|
||||
public String adapt(Object javaObject) {
|
||||
if (javaObject instanceof String) {
|
||||
return (String) javaObject;
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to String");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toJava(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject repr = PyObject_Str(pythonObject.getNativePythonObject());
|
||||
PyObject str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~");
|
||||
String jstr = PyBytes_AsString(str).getString();
|
||||
Py_DecRef(repr);
|
||||
Py_DecRef(str);
|
||||
return jstr;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(String javaObject) {
|
||||
return new PythonObject(PyUnicode_FromString(javaObject));
|
||||
}
|
||||
};
|
||||
|
||||
public static final PythonType<Long> INT = new PythonType<Long>("int", Long.class) {
|
||||
@Override
|
||||
public Long adapt(Object javaObject) {
|
||||
if (javaObject instanceof Number) {
|
||||
return ((Number) javaObject).longValue();
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Long");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long toJava(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
long val = PyLong_AsLong(pythonObject.getNativePythonObject());
|
||||
if (val == -1 && PyErr_Occurred() != null) {
|
||||
throw new PythonException("Could not convert value to int: " + pythonObject.toString());
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accepts(Object javaObject) {
|
||||
return (javaObject instanceof Integer) || (javaObject instanceof Long);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(Long javaObject) {
|
||||
return new PythonObject(PyLong_FromLong(javaObject));
|
||||
}
|
||||
};
|
||||
|
||||
public static final PythonType<Double> FLOAT = new PythonType<Double>("float", Double.class) {
|
||||
|
||||
@Override
|
||||
public Double adapt(Object javaObject) {
|
||||
if (javaObject instanceof Number) {
|
||||
return ((Number) javaObject).doubleValue();
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Long");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Double toJava(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
double val = PyFloat_AsDouble(pythonObject.getNativePythonObject());
|
||||
if (val == -1 && PyErr_Occurred() != null) {
|
||||
throw new PythonException("Could not convert value to float: " + pythonObject.toString());
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accepts(Object javaObject) {
|
||||
return (javaObject instanceof Float) || (javaObject instanceof Double);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(Double javaObject) {
|
||||
return new PythonObject(PyFloat_FromDouble(javaObject));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
public static final PythonType<Boolean> BOOL = new PythonType<Boolean>("bool", Boolean.class) {
|
||||
|
||||
@Override
|
||||
public Boolean adapt(Object javaObject) {
|
||||
if (javaObject instanceof Boolean) {
|
||||
return (Boolean) javaObject;
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Boolean");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean toJava(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject builtins = PyImport_ImportModule("builtins");
|
||||
PyObject boolF = PyObject_GetAttrString(builtins, "bool");
|
||||
|
||||
PythonObject bool = new PythonObject(boolF, false).call(pythonObject);
|
||||
boolean ret = PyLong_AsLong(bool.getNativePythonObject()) > 0;
|
||||
bool.del();
|
||||
Py_DecRef(boolF);
|
||||
Py_DecRef(builtins);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(Boolean javaObject) {
|
||||
return new PythonObject(PyBool_FromLong(javaObject ? 1 : 0));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
public static final PythonType<List> LIST = new PythonType<List>("list", List.class) {
|
||||
|
||||
@Override
|
||||
public boolean accepts(Object javaObject) {
|
||||
return (javaObject instanceof List || javaObject.getClass().isArray());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List adapt(Object javaObject) {
|
||||
if (javaObject instanceof List) {
|
||||
return (List) javaObject;
|
||||
} else if (javaObject.getClass().isArray()) {
|
||||
List<Object> ret = new ArrayList<>();
|
||||
if (javaObject instanceof Object[]) {
|
||||
Object[] arr = (Object[]) javaObject;
|
||||
return new ArrayList<>(Arrays.asList(arr));
|
||||
} else if (javaObject instanceof short[]) {
|
||||
short[] arr = (short[]) javaObject;
|
||||
for (short x : arr) ret.add(x);
|
||||
return ret;
|
||||
} else if (javaObject instanceof int[]) {
|
||||
int[] arr = (int[]) javaObject;
|
||||
for (int x : arr) ret.add(x);
|
||||
return ret;
|
||||
}else if (javaObject instanceof byte[]){
|
||||
byte[] arr = (byte[]) javaObject;
|
||||
for (int x : arr) ret.add(x & 0xff);
|
||||
return ret;
|
||||
} else if (javaObject instanceof long[]) {
|
||||
long[] arr = (long[]) javaObject;
|
||||
for (long x : arr) ret.add(x);
|
||||
return ret;
|
||||
} else if (javaObject instanceof float[]) {
|
||||
float[] arr = (float[]) javaObject;
|
||||
for (float x : arr) ret.add(x);
|
||||
return ret;
|
||||
} else if (javaObject instanceof double[]) {
|
||||
double[] arr = (double[]) javaObject;
|
||||
for (double x : arr) ret.add(x);
|
||||
return ret;
|
||||
} else if (javaObject instanceof boolean[]) {
|
||||
boolean[] arr = (boolean[]) javaObject;
|
||||
for (boolean x : arr) ret.add(x);
|
||||
return ret;
|
||||
} else {
|
||||
throw new PythonException("Unsupported array type: " + javaObject.getClass().toString());
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to List");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List toJava(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
List ret = new ArrayList();
|
||||
long n = PyObject_Size(pythonObject.getNativePythonObject());
|
||||
if (n < 0) {
|
||||
throw new PythonException("Object cannot be interpreted as a List");
|
||||
}
|
||||
for (long i = 0; i < n; i++) {
|
||||
PyObject pyIndex = PyLong_FromLong(i);
|
||||
PyObject pyItem = PyObject_GetItem(pythonObject.getNativePythonObject(),
|
||||
pyIndex);
|
||||
Py_DecRef(pyIndex);
|
||||
PythonType pyItemType = getPythonTypeForPythonObject(new PythonObject(pyItem, false));
|
||||
ret.add(pyItemType.toJava(new PythonObject(pyItem, false)));
|
||||
Py_DecRef(pyItem);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(List javaObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject pyList = PyList_New(javaObject.size());
|
||||
for (int i = 0; i < javaObject.size(); i++) {
|
||||
Object item = javaObject.get(i);
|
||||
PythonObject pyItem;
|
||||
boolean owned;
|
||||
if (item instanceof PythonObject) {
|
||||
pyItem = (PythonObject) item;
|
||||
owned = false;
|
||||
} else if (item instanceof PyObject) {
|
||||
pyItem = new PythonObject((PyObject) item, false);
|
||||
owned = false;
|
||||
} else {
|
||||
pyItem = PythonTypes.convert(item);
|
||||
owned = true;
|
||||
}
|
||||
Py_IncRef(pyItem.getNativePythonObject()); // reference will be stolen by PyList_SetItem()
|
||||
PyList_SetItem(pyList, i, pyItem.getNativePythonObject());
|
||||
if (owned) pyItem.del();
|
||||
}
|
||||
return new PythonObject(pyList);
|
||||
}
|
||||
};
|
||||
|
||||
public static final PythonType<Map> DICT = new PythonType<Map>("dict", Map.class) {
|
||||
|
||||
@Override
|
||||
public Map adapt(Object javaObject) {
|
||||
if (javaObject instanceof Map) {
|
||||
return (Map) javaObject;
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Map");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map toJava(PythonObject pythonObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
HashMap ret = new HashMap();
|
||||
PyObject dictType = new PyObject(PyDict_Type());
|
||||
if (PyObject_IsInstance(pythonObject.getNativePythonObject(), dictType) != 1) {
|
||||
throw new PythonException("Expected dict, received: " + pythonObject.toString());
|
||||
}
|
||||
|
||||
PyObject keys = PyDict_Keys(pythonObject.getNativePythonObject());
|
||||
PyObject keysIter = PyObject_GetIter(keys);
|
||||
PyObject vals = PyDict_Values(pythonObject.getNativePythonObject());
|
||||
PyObject valsIter = PyObject_GetIter(vals);
|
||||
try {
|
||||
long n = PyObject_Size(pythonObject.getNativePythonObject());
|
||||
for (long i = 0; i < n; i++) {
|
||||
PythonObject pyKey = new PythonObject(PyIter_Next(keysIter), false);
|
||||
PythonObject pyVal = new PythonObject(PyIter_Next(valsIter), false);
|
||||
PythonType pyKeyType = getPythonTypeForPythonObject(pyKey);
|
||||
PythonType pyValType = getPythonTypeForPythonObject(pyVal);
|
||||
ret.put(pyKeyType.toJava(pyKey), pyValType.toJava(pyVal));
|
||||
Py_DecRef(pyKey.getNativePythonObject());
|
||||
Py_DecRef(pyVal.getNativePythonObject());
|
||||
}
|
||||
} finally {
|
||||
Py_DecRef(keysIter);
|
||||
Py_DecRef(valsIter);
|
||||
Py_DecRef(keys);
|
||||
Py_DecRef(vals);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(Map javaObject) {
|
||||
PythonGIL.assertThreadSafe();
|
||||
PyObject pyDict = PyDict_New();
|
||||
for (Object k : javaObject.keySet()) {
|
||||
PythonObject pyKey;
|
||||
if (k instanceof PythonObject) {
|
||||
pyKey = (PythonObject) k;
|
||||
} else if (k instanceof PyObject) {
|
||||
pyKey = new PythonObject((PyObject) k);
|
||||
} else {
|
||||
pyKey = PythonTypes.convert(k);
|
||||
}
|
||||
Object v = javaObject.get(k);
|
||||
PythonObject pyVal;
|
||||
if (v instanceof PythonObject) {
|
||||
pyVal = (PythonObject) v;
|
||||
} else if (v instanceof PyObject) {
|
||||
pyVal = new PythonObject((PyObject) v);
|
||||
} else {
|
||||
pyVal = PythonTypes.convert(v);
|
||||
}
|
||||
int errCode = PyDict_SetItem(pyDict, pyKey.getNativePythonObject(), pyVal.getNativePythonObject());
|
||||
if (errCode != 0) {
|
||||
String keyStr = pyKey.toString();
|
||||
pyKey.del();
|
||||
pyVal.del();
|
||||
throw new PythonException("Unable to create python dictionary. Unhashable key: " + keyStr);
|
||||
}
|
||||
pyKey.del();
|
||||
pyVal.del();
|
||||
}
|
||||
return new PythonObject(pyDict);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
public static final PythonType<byte[]> BYTES = new PythonType<byte[]>("bytes", byte[].class) {
|
||||
@Override
|
||||
public byte[] toJava(PythonObject pythonObject) {
|
||||
try (PythonGC gc = PythonGC.watch()) {
|
||||
if (!(Python.isinstance(pythonObject, Python.bytesType()))) {
|
||||
throw new PythonException("Expected bytes. Received: " + pythonObject);
|
||||
}
|
||||
PythonObject pySize = Python.len(pythonObject);
|
||||
byte[] ret = new byte[pySize.toInt()];
|
||||
for (int i = 0; i < ret.length; i++) {
|
||||
ret[i] = (byte)pythonObject.get(i).toInt();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(byte[] javaObject) {
|
||||
try(PythonGC gc = PythonGC.watch()){
|
||||
PythonObject ret = Python.bytes(LIST.toPython(LIST.adapt(javaObject)));
|
||||
PythonGC.keep(ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public boolean accepts(Object javaObject) {
|
||||
return javaObject instanceof byte[];
|
||||
}
|
||||
@Override
|
||||
public byte[] adapt(Object javaObject) {
|
||||
if (javaObject instanceof byte[]){
|
||||
return (byte[])javaObject;
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to byte[]");
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
@lombok.Data
|
||||
public class PythonVariable<T> {
|
||||
|
||||
private String name;
|
||||
private String type;
|
||||
private T value;
|
||||
|
||||
private static boolean validateVariableName(String s) {
|
||||
if (s.isEmpty()) return false;
|
||||
if (!Character.isJavaIdentifierStart(s.charAt(0))) return false;
|
||||
for (int i = 1; i < s.length(); i++)
|
||||
if (!Character.isJavaIdentifierPart(s.charAt(i)))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
public PythonVariable(String name, PythonType<T> type, Object value) {
|
||||
if (!validateVariableName(name)) {
|
||||
throw new PythonException("Invalid identifier: " + name);
|
||||
}
|
||||
this.name = name;
|
||||
this.type = type.getName();
|
||||
setValue(value);
|
||||
}
|
||||
|
||||
public PythonVariable(String name, PythonType<T> type) {
|
||||
this(name, type, null);
|
||||
}
|
||||
|
||||
public PythonType<T> getType() {
|
||||
return PythonTypes.get(this.type);
|
||||
}
|
||||
|
||||
public T getValue() {
|
||||
return this.value;
|
||||
}
|
||||
|
||||
public void setValue(Object value) {
|
||||
this.value = value == null ? null : getType().adapt(value);
|
||||
}
|
||||
|
||||
public PythonObject getPythonObject() {
|
||||
return getType().toPython(value);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Some syntax sugar for lookup by name
|
||||
*/
|
||||
public class PythonVariables extends ArrayList<PythonVariable> {
|
||||
public PythonVariable get(String variableName) {
|
||||
for (PythonVariable pyVar: this){
|
||||
if (pyVar.getName().equals(variableName)){
|
||||
return pyVar;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public <T> boolean add(String variableName, PythonType<T> variableType, Object value){
|
||||
return this.add(new PythonVariable<>(variableName, variableType, value));
|
||||
}
|
||||
|
||||
public PythonVariables(PythonVariable... variables){
|
||||
this(Arrays.asList(variables));
|
||||
}
|
||||
public PythonVariables(List<PythonVariable> list){
|
||||
super();
|
||||
addAll(list);
|
||||
}
|
||||
}
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
package org.nd4j.python4j;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
import org.nd4j.common.io.ClassPathResource;
|
||||
import org.nd4j.common.primitives.Pair;
|
||||
import org.nd4j.shade.netty.util.concurrent.FastThreadLocal;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* A port of UncheckedPythonWrapper from:
|
||||
* https://github.com/invesdwin/invesdwin-context-python/blob/master/invesdwin-context-python-parent/invesdwin-context-python-runtime-python4j/src/main/java/de/invesdwin/context/python/runtime/python4j/internal/UncheckedPythonEngineWrapper.java#L125
|
||||
*
|
||||
*
|
||||
*/
|
||||
public class UncheckedPythonInterpreter implements PythonInterpreter {
|
||||
|
||||
|
||||
private static final String ANS = "__ans__";
|
||||
private static final String ANS_EQUALS = ANS + " = ";
|
||||
|
||||
private static final FastThreadLocal<UncheckedPythonInterpreter> INSTANCE;
|
||||
|
||||
private static PyObject globals;
|
||||
private static PyObject globalsAns;
|
||||
|
||||
private final static Map<String, Pair<PythonObject,Object>> cachedVariables = new ConcurrentHashMap<>();
|
||||
|
||||
private final GILLock gilLock = new GILLock();
|
||||
|
||||
|
||||
|
||||
static {
|
||||
|
||||
INSTANCE = new FastThreadLocal<UncheckedPythonInterpreter>() {
|
||||
@Override
|
||||
protected UncheckedPythonInterpreter initialValue() throws Exception {
|
||||
return new UncheckedPythonInterpreter();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
private UncheckedPythonInterpreter() {
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Get the cached python object.
|
||||
* @param varName the name of the variable
|
||||
* @return the cached object
|
||||
*/
|
||||
@Override
|
||||
public Object getCachedPython(String varName) {
|
||||
return cachedVariables.get(varName).getKey();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the cached java objects.
|
||||
* @param varName the name of the variable
|
||||
* @return the cached object
|
||||
*/
|
||||
@Override
|
||||
public Object getCachedJava(String varName) {
|
||||
return cachedVariables.get(varName).getValue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the cached python/java objects.
|
||||
* @param varName the name of the variable
|
||||
* @return the cached object
|
||||
*/
|
||||
@Override
|
||||
public Pair<PythonObject,Object> getCachedPythonJava(String varName) {
|
||||
return cachedVariables.get(varName);
|
||||
}
|
||||
|
||||
public PythonObject newNone() {
|
||||
evalUnchecked(ANS_EQUALS + "None");
|
||||
return getAns();
|
||||
}
|
||||
|
||||
public void init() {
|
||||
synchronized (UncheckedPythonInterpreter.class) {
|
||||
if (UncheckedPythonInterpreter.globals != null) {
|
||||
return;
|
||||
}
|
||||
|
||||
gilLock.lock();
|
||||
PythonExecutioner.init();
|
||||
|
||||
try (InputStream is = new ClassPathResource(UncheckedPythonInterpreter.class.getSimpleName() + ".py").getInputStream()) {
|
||||
String code = IOUtils.toString(is, Charset.defaultCharset());
|
||||
final int result = org.bytedeco.cpython.global.python.PyRun_SimpleString(code);
|
||||
if (result != 0) {
|
||||
throw new PythonException("Execution failed, unable to retrieve python exception.");
|
||||
}
|
||||
|
||||
|
||||
final PyObject main = org.bytedeco.cpython.global.python.PyImport_ImportModule("__main__");
|
||||
UncheckedPythonInterpreter.globals = org.bytedeco.cpython.global.python.PyModule_GetDict(main);
|
||||
UncheckedPythonInterpreter.globalsAns = org.bytedeco.cpython.global.python.PyUnicode_FromString(ANS);
|
||||
//we keep the refs eternally
|
||||
//org.bytedeco.cpython.global.python.Py_DecRef(main);
|
||||
//org.bytedeco.cpython.global.python.Py_DecRef(globals);
|
||||
//org.bytedeco.cpython.global.python.Py_DecRef(globalsAns);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
gilLock.unlock();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void eval(final String expression) {
|
||||
gilLock.lock();
|
||||
try {
|
||||
evalUnchecked(expression);
|
||||
} finally {
|
||||
gilLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void evalUnchecked(final String expression) {
|
||||
final int result = org.bytedeco.cpython.global.python.PyRun_SimpleString(expression);
|
||||
if (result != 0) {
|
||||
throw new PythonException("Execution failed, unable to retrieve python exception.");
|
||||
}
|
||||
}
|
||||
|
||||
private PythonObject getAns() {
|
||||
return new PythonObject(org.bytedeco.cpython.global.python.PyObject_GetItem(globals, globalsAns), false);
|
||||
}
|
||||
|
||||
|
||||
public static UncheckedPythonInterpreter getInstance() {
|
||||
return UncheckedPythonInterpreter.INSTANCE.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public GILLock gilLock() {
|
||||
return gilLock;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec(String expression) {
|
||||
eval(expression);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object get(String variable, boolean getNew) {
|
||||
Object ret = null;
|
||||
gilLock.lock();
|
||||
try {
|
||||
evalUnchecked(ANS_EQUALS + variable);
|
||||
final PythonObject ans = getAns();
|
||||
final PythonType<Object> type = PythonTypes.getPythonTypeForPythonObject(ans);
|
||||
Object o = type.toJava(ans);
|
||||
cachedVariables.put(variable,Pair.of(ans,o));
|
||||
ret = o;
|
||||
|
||||
} finally {
|
||||
gilLock.unlock();
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void set(String variable, Object value) {
|
||||
gilLock.lock();
|
||||
try {
|
||||
if (value == null) {
|
||||
evalUnchecked(variable + " = None");
|
||||
cachedVariables.put(variable,Pair.of(UncheckedPythonInterpreter.getInstance().newNone(),null));
|
||||
} else {
|
||||
final PythonObject converted = PythonTypes.convert(value);
|
||||
org.bytedeco.cpython.global.python.PyDict_SetItemString(globals, variable,
|
||||
converted.getNativePythonObject());
|
||||
cachedVariables.put(variable,Pair.of(converted,value));
|
||||
}
|
||||
} catch (final Throwable t) {
|
||||
throw new RuntimeException("Variable=" + variable + " Value=" + value, t);
|
||||
} finally {
|
||||
gilLock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
open module python4j.core {
|
||||
requires commons.io;
|
||||
requires nd4j.common;
|
||||
requires org.bytedeco.javacpp;
|
||||
requires slf4j.api;
|
||||
requires org.bytedeco.cpython;
|
||||
exports org.nd4j.python4j;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
import sys
|
||||
|
||||
# Based on work from: https://github.com/invesdwin/invesdwin-context-python/blob/master/invesdwin-context-python-parent/invesdwin-context-python-runtime-python4j/src/main/java/de/invesdwin/context/python/runtime/python4j/internal/UncheckedPythonEngineWrapper.py
|
||||
|
||||
# http://stackoverflow.com/questions/3543833/how-do-i-clear-all-variables-in-the-middle-of-a-python-script
|
||||
# Permission for use granted here: https://github.com/eclipse/deeplearning4j/issues/9595
|
||||
__saved_context__ = {}
|
||||
|
||||
def saveContext():
|
||||
__saved_context__.update(sys.modules[__name__].__dict__)
|
||||
|
||||
def restoreContext():
|
||||
names = list(sys.modules[__name__].__dict__.keys())
|
||||
for n in names:
|
||||
if n not in __saved_context__:
|
||||
del sys.modules[__name__].__dict__[n]
|
||||
|
||||
saveContext()
|
||||
@@ -0,0 +1,18 @@
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
import json
|
||||
import inspect
|
||||
|
||||
__python_exception__ = ""
|
||||
try:
|
||||
pass
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
except Exception as ex:
|
||||
__python_exception__ = ex
|
||||
try:
|
||||
exc_info = sys.exc_info()
|
||||
finally:
|
||||
print(ex)
|
||||
traceback.print_exception(*exc_info)
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ /* ******************************************************************************
|
||||
~ *
|
||||
~ *
|
||||
~ * This program and the accompanying materials are made available under the
|
||||
~ * terms of the Apache License, Version 2.0 which is available at
|
||||
~ * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
~ *
|
||||
~ * See the NOTICE file distributed with this work for additional
|
||||
~ * information regarding copyright ownership.
|
||||
~ * Unless required by applicable law or agreed to in writing, software
|
||||
~ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
~ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
~ * License for the specific language governing permissions and limitations
|
||||
~ * under the License.
|
||||
~ *
|
||||
~ * SPDX-License-Identifier: Apache-2.0
|
||||
~ ******************************************************************************/
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>python4j-parent</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>python4j-numpy</artifactId>
|
||||
|
||||
<properties>
|
||||
<module.name>python4j.numpy</module.name>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.moditect</groupId>
|
||||
<artifactId>moditect-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.bytedeco</groupId>
|
||||
<artifactId>numpy-platform</artifactId>
|
||||
<version>${numpy.javacpp.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native-api</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>python4j-core</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,336 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
|
||||
package org.nd4j.python4j.numpy;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.bytedeco.cpython.PyObject;
|
||||
import org.bytedeco.cpython.PyTypeObject;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
import org.bytedeco.javacpp.SizeTPointer;
|
||||
import org.bytedeco.numpy.PyArrayObject;
|
||||
import org.bytedeco.numpy.global.numpy;
|
||||
import org.nd4j.linalg.api.buffer.DataBuffer;
|
||||
import org.nd4j.linalg.api.buffer.DataType;
|
||||
import org.nd4j.linalg.api.concurrency.AffinityManager;
|
||||
import org.nd4j.linalg.api.memory.MemoryWorkspace;
|
||||
import org.nd4j.linalg.api.ndarray.INDArray;
|
||||
import org.nd4j.linalg.api.shape.Shape;
|
||||
import org.nd4j.linalg.factory.Nd4j;
|
||||
import org.nd4j.nativeblas.NativeOpsHolder;
|
||||
import org.nd4j.python4j.*;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import static org.bytedeco.cpython.global.python.*;
|
||||
import static org.bytedeco.numpy.global.numpy.*;
|
||||
|
||||
@Slf4j
|
||||
public class NumpyArray extends PythonType<INDArray> {
|
||||
|
||||
public static final NumpyArray INSTANCE;
|
||||
private static final AtomicBoolean init = new AtomicBoolean(false);
|
||||
private static final Map<String, DataBuffer> cache = new HashMap<>();
|
||||
public final static String IMPORT_NUMPY_ARRAY = "org.eclipse.python4j.numpyimport";
|
||||
public final static String ADD_JAVACPP_NUMPY_TO_PATH = "org.eclipse.python4j.numpyimport";
|
||||
public final static String DEFAULT_IMPORT_NUMPY_ARRAY = "true";
|
||||
public final static String DEFAULT_ADD_JAVACPP_NUMPY_TO_PATH = "true";
|
||||
|
||||
static {
|
||||
new PythonExecutioner();
|
||||
INSTANCE = new NumpyArray();
|
||||
INSTANCE.init();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public File[] packages(){
|
||||
try{
|
||||
return new File[]{numpy.cachePackage()};
|
||||
}catch(Exception e){
|
||||
throw new PythonException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public synchronized void init() {
|
||||
if (init.get()) return;
|
||||
init.set(true);
|
||||
|
||||
if(Boolean.parseBoolean(System.getProperty(IMPORT_NUMPY_ARRAY,DEFAULT_IMPORT_NUMPY_ARRAY))) {
|
||||
//See: https://numpy.org/doc/1.17/reference/c-api.array.html#importing-the-api
|
||||
//DO NOT REMOVE
|
||||
if(Boolean.parseBoolean(System.getProperty(ADD_JAVACPP_NUMPY_TO_PATH,DEFAULT_ADD_JAVACPP_NUMPY_TO_PATH))) {
|
||||
for(File packageDir : numpy.cachePackages()) {
|
||||
PyObject pythonPath = PyUnicode_FromString(packageDir.getAbsolutePath());
|
||||
PySys_SetObject("path", pythonPath);
|
||||
Py_DecRef(pythonPath);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//ensure python doesn't get initialized twice, this call is needed before numpy import array
|
||||
PythonConstants.setInitializePython(false);
|
||||
Py_Initialize();
|
||||
|
||||
int err = numpy._import_array();
|
||||
if (err < 0){
|
||||
System.out.println("Numpy import failed!");
|
||||
throw new PythonException("Numpy import failed!");
|
||||
}
|
||||
}
|
||||
|
||||
if (PythonGIL.locked()) {
|
||||
throw new PythonException("Can not initialize numpy - GIL already acquired.");
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
public NumpyArray() {
|
||||
super("numpy.ndarray", INDArray.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public INDArray toJava(PythonObject pythonObject) {
|
||||
log.debug("Converting PythonObject to INDArray...");
|
||||
PyObject np = PyImport_ImportModule("numpy");
|
||||
PyObject ndarray = PyObject_GetAttrString(np, "ndarray");
|
||||
if (PyObject_IsInstance(pythonObject.getNativePythonObject(), ndarray) != 1) {
|
||||
Py_DecRef(ndarray);
|
||||
Py_DecRef(np);
|
||||
throw new PythonException("Object is not a numpy array! Use Python.ndarray() to convert object to a numpy array.");
|
||||
}
|
||||
Py_DecRef(ndarray);
|
||||
Py_DecRef(np);
|
||||
PyArrayObject npArr = new PyArrayObject(pythonObject.getNativePythonObject());
|
||||
long[] shape = new long[PyArray_NDIM(npArr)];
|
||||
SizeTPointer shapePtr = PyArray_SHAPE(npArr);
|
||||
if (shapePtr != null)
|
||||
shapePtr.get(shape, 0, shape.length);
|
||||
long[] strides = new long[shape.length];
|
||||
SizeTPointer stridesPtr = PyArray_STRIDES(npArr);
|
||||
if (stridesPtr != null)
|
||||
stridesPtr.get(strides, 0, strides.length);
|
||||
int npdtype = PyArray_TYPE(npArr);
|
||||
|
||||
DataType dtype;
|
||||
switch (npdtype) {
|
||||
case NPY_DOUBLE:
|
||||
dtype = DataType.DOUBLE;
|
||||
break;
|
||||
case NPY_FLOAT:
|
||||
dtype = DataType.FLOAT;
|
||||
break;
|
||||
case NPY_SHORT:
|
||||
dtype = DataType.SHORT;
|
||||
break;
|
||||
case NPY_INT:
|
||||
dtype = DataType.INT32;
|
||||
break;
|
||||
case NPY_LONG:
|
||||
dtype = DataType.INT64;
|
||||
break;
|
||||
case NPY_UINT:
|
||||
dtype = DataType.UINT32;
|
||||
break;
|
||||
case NPY_BYTE:
|
||||
dtype = DataType.INT8;
|
||||
break;
|
||||
case NPY_UBYTE:
|
||||
dtype = DataType.UINT8;
|
||||
break;
|
||||
case NPY_BOOL:
|
||||
dtype = DataType.BOOL;
|
||||
break;
|
||||
case NPY_HALF:
|
||||
dtype = DataType.FLOAT16;
|
||||
break;
|
||||
case NPY_LONGLONG:
|
||||
dtype = DataType.INT64;
|
||||
break;
|
||||
case NPY_USHORT:
|
||||
dtype = DataType.UINT16;
|
||||
break;
|
||||
case NPY_ULONG:
|
||||
case NPY_ULONGLONG:
|
||||
dtype = DataType.UINT64;
|
||||
break;
|
||||
default:
|
||||
throw new PythonException("Unsupported array data type: " + npdtype);
|
||||
}
|
||||
long size = 1;
|
||||
for (int i = 0; i < shape.length; size *= shape[i++]) ;
|
||||
|
||||
INDArray ret;
|
||||
long address = PyArray_DATA(npArr).address();
|
||||
String key = address + "_" + size + "_" + dtype;
|
||||
DataBuffer buff = cache.get(key);
|
||||
if (buff == null) {
|
||||
try (MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
|
||||
Pointer ptr = Nd4j.getNativeOps().pointerForAddress(address);
|
||||
ptr = ptr.limit(size);
|
||||
ptr = ptr.capacity(size);
|
||||
buff = Nd4j.createBuffer(ptr, size, dtype);
|
||||
cache.put(key, buff);
|
||||
}
|
||||
}
|
||||
int elemSize = buff.getElementSize();
|
||||
long[] nd4jStrides = new long[strides.length];
|
||||
for (int i = 0; i < strides.length; i++) {
|
||||
nd4jStrides[i] = strides[i] / elemSize;
|
||||
}
|
||||
ret = Nd4j.create(buff, shape, nd4jStrides, 0, Shape.getOrder(shape, nd4jStrides, 1), dtype);
|
||||
Nd4j.getAffinityManager().tagLocation(ret, AffinityManager.Location.HOST);
|
||||
log.debug("Done creating numpy array.");
|
||||
return ret;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public PythonObject toPython(INDArray indArray) {
|
||||
log.debug("Converting INDArray to PythonObject...");
|
||||
DataType dataType = indArray.dataType();
|
||||
DataBuffer buff = indArray.data();
|
||||
String key = buff.pointer().address() + "_" + buff.length() + "_" + dataType;
|
||||
cache.put(key, buff);
|
||||
int numpyType;
|
||||
String ctype;
|
||||
switch (dataType) {
|
||||
case DOUBLE:
|
||||
numpyType = NPY_DOUBLE;
|
||||
ctype = "c_double";
|
||||
break;
|
||||
case FLOAT:
|
||||
case BFLOAT16:
|
||||
numpyType = NPY_FLOAT;
|
||||
ctype = "c_float";
|
||||
break;
|
||||
case SHORT:
|
||||
numpyType = NPY_SHORT;
|
||||
ctype = "c_short";
|
||||
break;
|
||||
case INT:
|
||||
numpyType = NPY_INT;
|
||||
ctype = "c_int";
|
||||
break;
|
||||
case LONG:
|
||||
numpyType = NPY_INT64;
|
||||
ctype = "c_int64";
|
||||
break;
|
||||
case UINT16:
|
||||
numpyType = NPY_USHORT;
|
||||
ctype = "c_uint16";
|
||||
break;
|
||||
case UINT32:
|
||||
numpyType = NPY_UINT;
|
||||
ctype = "c_uint";
|
||||
break;
|
||||
case UINT64:
|
||||
numpyType = NPY_UINT64;
|
||||
ctype = "c_uint64";
|
||||
break;
|
||||
case BOOL:
|
||||
numpyType = NPY_BOOL;
|
||||
ctype = "c_bool";
|
||||
break;
|
||||
case BYTE:
|
||||
numpyType = NPY_BYTE;
|
||||
ctype = "c_byte";
|
||||
break;
|
||||
case UBYTE:
|
||||
numpyType = NPY_UBYTE;
|
||||
ctype = "c_ubyte";
|
||||
break;
|
||||
case HALF:
|
||||
numpyType = NPY_HALF;
|
||||
ctype = "c_short";
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Unsupported dtype: " + dataType);
|
||||
}
|
||||
|
||||
long[] shape = indArray.shape();
|
||||
INDArray inputArray = indArray;
|
||||
if (dataType == DataType.BFLOAT16) {
|
||||
log.warn("Creating copy of array as bfloat16 is not supported by numpy.");
|
||||
inputArray = indArray.castTo(DataType.FLOAT);
|
||||
}
|
||||
|
||||
//Sync to host memory in the case of CUDA, before passing the host memory pointer to Python
|
||||
|
||||
Nd4j.getAffinityManager().ensureLocation(inputArray, AffinityManager.Location.HOST);
|
||||
|
||||
// PyArray_Type() call causes jvm crash in linux cpu if GIL is acquired by non main thread.
|
||||
// Using Interpreter for now:
|
||||
|
||||
//likely embedded in python, always use this method instead
|
||||
if(!PythonConstants.releaseGilAutomatically() || PythonConstants.createNpyViaPython()) {
|
||||
try(PythonContextManager.Context context = new PythonContextManager.Context("__np_array_converter")){
|
||||
log.debug("Stringing exec...");
|
||||
String code = "import ctypes\nimport numpy as np\n" +
|
||||
"cArr = (ctypes." + ctype + "*" + indArray.length() + ")"+
|
||||
".from_address(" + indArray.data().pointer().address() + ")\n"+
|
||||
"npArr = np.frombuffer(cArr, dtype=" + ((numpyType == NPY_HALF) ? "'half'" : "ctypes." + ctype)+
|
||||
").reshape(" + Arrays.toString(indArray.shape()) + ")";
|
||||
PythonExecutioner.exec(code);
|
||||
log.debug("exec done.");
|
||||
PythonObject ret = PythonExecutioner.getVariable("npArr");
|
||||
Py_IncRef(ret.getNativePythonObject());
|
||||
return ret;
|
||||
|
||||
}
|
||||
} else {
|
||||
log.debug("NUMPY: PyArray_Type()");
|
||||
PyTypeObject pyTypeObject = PyArray_Type();
|
||||
|
||||
|
||||
log.debug("NUMPY: PyArray_New()");
|
||||
PyObject npArr = PyArray_New(pyTypeObject, shape.length, new SizeTPointer(shape),
|
||||
numpyType, null,
|
||||
inputArray.data().addressPointer(),
|
||||
0, NPY_ARRAY_CARRAY, null);
|
||||
log.debug("Created numpy array.");
|
||||
return new PythonObject(npArr);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accepts(Object javaObject) {
|
||||
return javaObject instanceof INDArray;
|
||||
}
|
||||
|
||||
@Override
|
||||
public INDArray adapt(Object javaObject) {
|
||||
if (javaObject instanceof INDArray) {
|
||||
return (INDArray) javaObject;
|
||||
}
|
||||
throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to INDArray");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
open module python4j.numpy {
|
||||
requires lombok;
|
||||
requires nd4j.cpu.api;
|
||||
requires org.bytedeco.cpython;
|
||||
requires org.bytedeco.javacpp;
|
||||
requires org.bytedeco.numpy;
|
||||
requires slf4j.api;
|
||||
requires nd4j.api;
|
||||
requires python4j.core;
|
||||
exports org.nd4j.python4j.numpy;
|
||||
provides org.nd4j.python4j.PythonType with org.nd4j.python4j.numpy.NumpyArray;
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# * Copyright (c) 2021 Deeplearning4j Contributors
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
org.nd4j.python4j.numpy.NumpyArray
|
||||
Reference in New Issue
Block a user