Massive Insert Data is very slow


#1

OrientDB Version: 3.0.17 (docker)
OS: debian 9
VPS, 10cores, 60gb RAM, 1600ssd.

In new database 500,000 records with duplicates added within 900 seconds.
I need speed is x1000. I have about 500GB raw data.
Is it really?
I creating DB from pairs: 1->2, 2->1, 1->3, 3->1, 2->1, 2>3, etc…

My code:

import com.orientechnologies.orient.core.db.ODatabaseSession;
import com.orientechnologies.orient.core.db.ODatabaseType;
import com.orientechnologies.orient.core.db.OrientDB;
import com.orientechnologies.orient.core.db.OrientDBConfig;
import com.orientechnologies.orient.core.intent.OIntentMassiveInsert;
import com.orientechnologies.orient.core.metadata.schema.OClass;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.OEdge;
import com.orientechnologies.orient.core.record.OVertex;
import com.orientechnologies.orient.core.sql.executor.OResult;
import com.orientechnologies.orient.core.sql.executor.OResultSet;

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.Optional;
import java.util.concurrent.TimeUnit;

public class Main {
    private static String[] getConfig() {
        String configFileName = "./config.txt";
        String source = "";
        String database = "";
        String databaseName = "";

        try {
            FileReader configFileReader = new FileReader(configFileName);
            BufferedReader readerConfig = new BufferedReader(configFileReader);

            source = readerConfig.readLine();
            database = readerConfig.readLine();
            databaseName = readerConfig.readLine();

            readerConfig.close();
        } catch (Exception e) {
            System.err.format("Exception occurred trying to read '%s'.", configFileName);
            e.printStackTrace();
        }


        String[] res = {source, database, databaseName};

        return res;
    }

    private static OVertex createPerson(ODatabaseSession db, String uid) {
        String statement = "SELECT FROM Person WHERE uid = ? LIMIT 1";
        OResultSet rs = db.query(statement, uid);

        if (rs.hasNext()) {
            OResult row = rs.next();
            Optional<OVertex> p = row.getVertex();
            if (p.isPresent()) {
                return p.get();
            } else {
                OVertex result = db.newVertex("Person");
                result.setProperty("uid", uid);
                result.save();

                return result;
            }
        } else {
            OVertex result = db.newVertex("Person");
            result.setProperty("uid", uid);
            result.save();

            return result;
        }
    }

    private static void createFriendOf(ODatabaseSession db, OVertex personFrom, OVertex personTo) {
        String statement = "SELECT FROM FriendOf WHERE out=? AND in=? LIMIT 1";
        OResultSet rs = db.query(statement, personFrom.getProperty("@rid"), personTo.getProperty("@rid"));

        if (rs.hasNext()) {
            return;
        } else {
            OEdge edge1 = personFrom.addEdge(personTo, "FriendOf");
            edge1.save();
        }
    }

    public static void main(String[] args) {
        String[] conf = getConfig();
        String source = conf[0];
        String database = conf[1];
        String databaseName = conf[2];

        OrientDB orient = new OrientDB(database, "root", "rootpwd", OrientDBConfig.defaultConfig());
//        boolean dbExists = orient.createIfNotExists(databaseName, ODatabaseType.PLOCAL);
        ODatabaseSession db = orient.open(databaseName, "admin", "admin");

//        System.out.println("dbExists " + dbExists);
//
//        if (!dbExists) {
//            OClass person = db.createVertexClass("Person");
//            person.createProperty("uid", OType.STRING);
//            person.createIndex("person_uid_index", OClass.INDEX_TYPE.UNIQUE, "uid");
//
//            OClass friendOf = db.createEdgeClass("FriendOf");
//            friendOf.createProperty("in", OType.LINK);
//            friendOf.createProperty("out", OType.LINK);
//            friendOf.createIndex("in_out", OClass.INDEX_TYPE.UNIQUE_HASH_INDEX, "out", "in");
//        }

        db.declareIntent(new OIntentMassiveInsert());

        int counter = 0;
        long startTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime());

        System.out.println("startTime " + startTime);

        try {
            FileReader fileReader = new FileReader(source);
            BufferedReader reader = new BufferedReader(fileReader);
            String line;
            String[] values;

            OVertex u1;
            OVertex u2;

            while ((line = reader.readLine()) != null) {
                values = line.split(",");

                u1 = createPerson(db, values[0]);
                u2 = createPerson(db, values[1]);
                createFriendOf(db, u1, u2);
                createFriendOf(db, u2, u1);

                counter++;

                if (counter % 10000 == 0) {
                    long end = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime());
                    System.out.println("count: " + counter + ", timeSpent:  " + (end - startTime));
                }
            }
            reader.close();
        } catch (Exception e) {
            System.err.format("Exception occurred trying to read '%s'.", source);
            e.printStackTrace();
        }

        db.declareIntent(null);

        long endTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime());

        System.out.println("count: " + counter);
        System.out.println("endTime " + endTime);
        orient.close();
        System.out.println("timeSpent:  " + (endTime - startTime));
    }
}

#2

Hi @209

do you have a dataset for running this code?

Are you running it wih remote or embedded?

Thanks