Search This Blog

Showing posts with label SQLFire. Show all posts
Showing posts with label SQLFire. Show all posts

Thursday, 5 September 2013

Pivotal SQLFire : Using -sync=false for Server nodes


Specifying -sync=false (the default for locators) causes the sqlf command to return control after the member reaches "waiting" state. With -sync=true (the default for servers), the sqlf command does not return control until after all dependent members have booted and the member has finished synchronizing disk stores.

Always use -sync=false when starting multiple members on the same machine, especially when executing sqlf commands from a shell script or batch file, so that the script file does not hang while waiting for a particular SQLFire member to start. You can use the sqlf locator wait and/or sqlf server wait later in the script to verify that each server has finished synchronizing and has reached the "running" state. 

Example Below:

Start Locator

sqlf locator start -peer-discovery-address=localhost -peer-discovery-port=41111 -conserve-sockets=false -client-bind-address=localhost -client-port=1527 -dir=locator -sync=false

Start Servers and run a script once system is up.

sqlf server start -server-groups=MYGROUP -client-bind-address=localhost -conserve-sockets=false -client-port=1528 -critical-heap-percentage=85 -eviction-heap-percentage=75 -locators=localhost[41111] -bind-address=localhost -dir=server1 -sync=false 

sqlf server start -server-groups=MYGROUP -client-bind-address=localhost -conserve-sockets=false -client-port=1529 -critical-heap-percentage=85 -eviction-heap-percentage=75 -locators=localhost[41111] -bind-address=localhost -dir=server2 -sync=false

sqlf locator wait -dir=locator
sqlf server wait -dir=server1
sqlf server wait -dir=server2

# Ready to run some SQL now system is up and running
sqlf <<!
connect client 'localhost:1527';
run './sql/add-list.sql';
!

Friday, 26 July 2013

Wednesday, 17 July 2013

Securing Schemas in Pivotal SQLFire


Authentication/authorization is off by default in SQLFire if servers have not specified it. That is why SQLFire connections don't need a user/password if not configured explicitly on server. Once authorization has been configured on server, users will have access to only their database objects. This is illustrated below. In this example we use PLAIN text for the passwords just to illustrate how we secure a schema

1. Create a locator script as follows

sqlf locator start -user=admin -password=adminpassword -sqlfire.sql-authorization=true -peer-discovery-address=Pas-Apicellas-MacBook-Pro.local -peer-discovery-port=41111 -client-bind-address=Pas-Apicellas-MacBook-Pro.local -client-port=1527 -dir=locator

2. Create a server start script as follows

sqlf server start $JVM_OPTS -user=admin -password=adminpassword -sqlfire.sql-authorization=true -max-heap=$MAXHEAP -initial-heap=$MINHEAP -critical-heap-percentage=85 -eviction-heap-percentage=75 -server-groups=MYGROUP -conserve-sockets=false -locators=Pas-Apicellas-MacBook-Pro.local[41111] -client-bind-address=Pas-Apicellas-MacBook-Pro.local -client-port=1528 -dir=server1

Note: Ensure that a sqlfire.properties file exists for the locator/server as shown below. 

sqlfire.properties

sqlfire.auth-provider=BUILTIN
sqlfire.sql-authorization=true
sqlfire.user.admin=adminpassword
sqlfire.user.locatoradmin=adminpassword
sqlfire.user.serveradmin=adminpassword

3. As the admin using connect with sqlf and create a database user account which will be the schema owner.
  
sqlf> connect client '127.0.0.1:1527;user=admin;password=adminpassword';
sqlf> CALL SYS.CREATE_USER('pas', 'pas');
Statement executed. 

4. Connect with a JDBC client as shown below.
  
private Connection getConnection() throws SQLException
 {
  Connection conn = null;
  conn = DriverManager.getConnection
      ("jdbc:sqlfire://127.0.0.1:1527/", "pas", "pas");
  return conn; 
 }

When the JDBC connection connects successfully it will then be in the context of the schema "pas" and will have full rights / access to that schema to be able to create database objects etc.

If we provide an invalid username/password we get an error as follows, hence securing access to the schema with a built in username we created at step #3 above.


Exception in thread "main" java.sql.SQLNonTransientConnectionException: Connection authentication failure occurred.  Reason: userid 'pas' or password invalid.


Thursday, 13 June 2013

Creating a multi threaded insert client for SQLFire 1.1

In this example below we show how to create a multi threaded insert client to insert 1,000,000 records into SQLFire table. In this example below the table is partitioned with synchronous persistence turned on.The distributed system includes one locator and 5 data members.

1. Create Table as shown below
  
drop diskstore store1;

CREATE DISKSTORE STORE1;

drop table person;

create table person 
(id int primary key,
 name varchar(40))
PARTITION BY COLUMN (id)
REDUNDANCY 1
PERSISTENT 'STORE1' SYNCHRONOUS;
2. Multi Threaded Insert client Code.
  
package com.pivotal.fe.test;

import java.io.IOException;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

public class MultiThreadInsert 
{
 private String url;
 private String driverClassName;
 private int RECORDS;
 private int COMMIT_POINT;
 private int nThreads;
 
 public MultiThreadInsert() throws IOException 
 {
  loadProperties();
 }
 
 public void loadProperties() throws IOException
 {
     Properties props = new Properties();
     URL propertiesUrl = ClassLoader.getSystemResource("sqlfiretest.properties");
     props.load(propertiesUrl.openStream());
     
     url = (String) props.getProperty("url");
     driverClassName = (String) props.getProperty("driverclassname");
     RECORDS = Integer.parseInt((String) props.getProperty("records"));
     COMMIT_POINT = Integer.parseInt((String) props.getProperty("commit_point"));
     nThreads = Integer.parseInt((String) props.getProperty("nThreads"));
     
 }
 
 @SuppressWarnings("unchecked")
 public void start() throws InterruptedException, SQLException 
 {
  System.out.printf("Starting %d threads for %d records connecting to %s\n", nThreads, RECORDS, url);
  
        final ExecutorService executorService = Executors.newFixedThreadPool(nThreads);

        ArrayList list = new ArrayList();
        for (int i = 0; i < nThreads; i++) {
            list.add(new RunData(url, driverClassName, i+1));
        }
        long start = System.currentTimeMillis();
        
        List<Future<?>> tasks = executorService.invokeAll(list, 5, TimeUnit.MINUTES);
        
        for(Future<?> f : tasks){
         try {
    f.get();
   } catch (ExecutionException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
   }
        }
        
     long end = System.currentTimeMillis() - start;
     
     float elapsedTimeSec = end/1000F;

        System.out.println(String.format("Elapsed time in seconds %f", elapsedTimeSec));
        
     executorService.shutdown();
        System.exit(0);
 }
 
 private class RunData implements Callable 
 {
     int counter = 0;
        int increment;
        Connection conn;
        String url;
        String driverClassName;
        
        private RunData(String url, String driverClassName, int increment) 
        {
            this.increment = increment;
            this.url = url;
            this.driverClassName = driverClassName;
        }

     public Connection getConnection() throws SQLException, ClassNotFoundException
     {
      Class.forName(driverClassName);
      Connection conn = null;
      conn = DriverManager.getConnection(url);
      //System.out.println("auto commit = " + conn.getAutoCommit());
      return conn; 
     }
     
        public void run() 
        {
      PreparedStatement stmt = null;
      String sql = "insert into person values (?, ?)";
      int counter = 0, size = 0;
      long startTime, endTime;
      
            int dataSize = RECORDS / nThreads;
            try 
            {
    conn = getConnection();
   } 
            catch (Exception e1) 
   {
    // TODO Auto-generated catch block
    e1.printStackTrace();
   }
            
            System.out.printf("Start: %d  End: %d \n",(dataSize * (increment - 1)), (dataSize * increment));
      try 
      {
       stmt = conn.prepareStatement(sql);
       
       startTime = System.currentTimeMillis();
       for (int i = (dataSize * (increment - 1)); i < (dataSize * increment); i++)
       {
        counter = counter + 1;
        size = size + 1;
        stmt.setInt(1, i);
        stmt.setString(2, "Person" + i);
        stmt.addBatch();
        
        if (counter % COMMIT_POINT == 0)
        {
         stmt.executeBatch();
         endTime = System.currentTimeMillis();
         System.out.printf("Insert batch of %d records took | %d | milliseconds\n", size, (endTime - startTime));
         startTime = System.currentTimeMillis();
         size = 0;
        }
       }
       
       /* there might be more records so call stmt.executeBatch() prior to commit here */
       stmt.executeBatch();
       
       //System.out.printf("Number of records submitted %d.\n", counter);
       
                 
      } 
   catch (SQLException e) 
   {
    // TODO Auto-generated catch block
    e.printStackTrace();
   }
   finally
   {
    if (stmt != null)
    {
     try 
     {
      stmt.close();
     } 
     catch (SQLException e) 
     {
      // TODO Auto-generated catch block
      e.printStackTrace();
     }
    }
   }      
        }
        
        public Object call() throws Exception 
        {
            run();
            return counter;
        }
  
 }

 /**
  * @param args
  * @throws InterruptedException 
  * @throws SQLException 
  * @throws IOException 
  */
 public static void main(String[] args) throws InterruptedException, SQLException, IOException 
 {
  // TODO Auto-generated method stub
  MultiThreadInsert test = new MultiThreadInsert();
  test.start();
 }
}
3. Output when run as follows.

Note: This was run on my MAC laptop which had 5 cache servers running on it. This would perform much better if I had 5 physical machines for each of the SQLFire cache server members.

[Fri Jul 05 22:33:47 papicella@:~/vmware/ant-demos/sqlfire/performance-test ] $ ant
Buildfile: /Users/papicella/vmware/ant-demos/sqlfire/performance-test/build.xml

init:

compile:

package:

run-insert-client:
     [echo] Starting insert client with jvm args : -server -showversion -Xms512m -Xmx512m
     [java] java version "1.6.0_37"
     [java] Java(TM) SE Runtime Environment (build 1.6.0_37-b06-434-11M3909)
     [java] Java HotSpot(TM) 64-Bit Server VM (build 20.12-b01-434, mixed mode)
     [java]
     [java] Starting 8 threads for 1000000 records connecting to jdbc:sqlfire://127.0.0.1:1527/;single-hop-enabled=true
     [java] Start: 875000  End: 1000000
     [java] Start: 125000  End: 250000
     [java] Start: 625000  End: 750000
     [java] Start: 500000  End: 625000
     [java] Start: 750000  End: 875000
     [java] Start: 375000  End: 500000
     [java] Start: 250000  End: 375000
     [java] Start: 0  End: 125000
     [java] Insert batch of 10000 records took | 1541 | milliseconds
     [java] Insert batch of 10000 records took | 1544 | milliseconds
     [java] Insert batch of 10000 records took | 1548 | milliseconds
     [java] Insert batch of 10000 records took | 1551 | milliseconds
     [java] Insert batch of 10000 records took | 1552 | milliseconds
     [java] Insert batch of 10000 records took | 1554 | milliseconds
     [java] Insert batch of 10000 records took | 1560 | milliseconds
     [java] Insert batch of 10000 records took | 1568 | milliseconds
     [java] Insert batch of 10000 records took | 552 | milliseconds
     [java] Insert batch of 10000 records took | 559 | milliseconds
     [java] Insert batch of 10000 records took | 563 | milliseconds
     [java] Insert batch of 10000 records took | 576 | milliseconds
     [java] Insert batch of 10000 records took | 581 | milliseconds
     [java] Insert batch of 10000 records took | 618 | milliseconds
     [java] Insert batch of 10000 records took | 600 | milliseconds
     [java] Insert batch of 10000 records took | 600 | milliseconds
     [java] Insert batch of 10000 records took | 630 | milliseconds
     [java] Insert batch of 10000 records took | 649 | milliseconds
     [java] Insert batch of 10000 records took | 643 | milliseconds
     [java] Insert batch of 10000 records took | 656 | milliseconds
     [java] Insert batch of 10000 records took | 687 | milliseconds
     [java] Insert batch of 10000 records took | 705 | milliseconds
     [java] Insert batch of 10000 records took | 705 | milliseconds
     [java] Insert batch of 10000 records took | 665 | milliseconds
     [java] Insert batch of 10000 records took | 682 | milliseconds
     [java] Insert batch of 10000 records took | 612 | milliseconds
     [java] Insert batch of 10000 records took | 712 | milliseconds
     [java] Insert batch of 10000 records took | 630 | milliseconds
     [java] Insert batch of 10000 records took | 646 | milliseconds
     [java] Insert batch of 10000 records took | 656 | milliseconds
     [java] Insert batch of 10000 records took | 661 | milliseconds
     [java] Insert batch of 10000 records took | 686 | milliseconds
     [java] Insert batch of 10000 records took | 475 | milliseconds
     [java] Insert batch of 10000 records took | 497 | milliseconds
     [java] Insert batch of 10000 records took | 499 | milliseconds
     [java] Insert batch of 10000 records took | 501 | milliseconds
     [java] Insert batch of 10000 records took | 534 | milliseconds
     [java] Insert batch of 10000 records took | 530 | milliseconds
     [java] Insert batch of 10000 records took | 534 | milliseconds
     [java] Insert batch of 10000 records took | 505 | milliseconds
     [java] Insert batch of 10000 records took | 534 | milliseconds
     [java] Insert batch of 10000 records took | 513 | milliseconds
     [java] Insert batch of 10000 records took | 535 | milliseconds
     [java] Insert batch of 10000 records took | 550 | milliseconds
     [java] Insert batch of 10000 records took | 504 | milliseconds
     [java] Insert batch of 10000 records took | 517 | milliseconds
     [java] Insert batch of 10000 records took | 528 | milliseconds
     [java] Insert batch of 10000 records took | 534 | milliseconds
     [java] Insert batch of 10000 records took | 523 | milliseconds
     [java] Insert batch of 10000 records took | 531 | milliseconds
     [java] Insert batch of 10000 records took | 517 | milliseconds
     [java] Insert batch of 10000 records took | 571 | milliseconds
     [java] Insert batch of 10000 records took | 598 | milliseconds
     [java] Insert batch of 10000 records took | 641 | milliseconds
     [java] Insert batch of 10000 records took | 652 | milliseconds
     [java] Insert batch of 10000 records took | 650 | milliseconds
     [java] Insert batch of 10000 records took | 498 | milliseconds
     [java] Insert batch of 10000 records took | 518 | milliseconds
     [java] Insert batch of 10000 records took | 542 | milliseconds
     [java] Insert batch of 10000 records took | 577 | milliseconds
     [java] Insert batch of 10000 records took | 557 | milliseconds
     [java] Insert batch of 10000 records took | 529 | milliseconds
     [java] Insert batch of 10000 records took | 550 | milliseconds
     [java] Insert batch of 10000 records took | 552 | milliseconds
     [java] Insert batch of 10000 records took | 525 | milliseconds
     [java] Insert batch of 10000 records took | 541 | milliseconds
     [java] Insert batch of 10000 records took | 544 | milliseconds
     [java] Insert batch of 10000 records took | 522 | milliseconds
     [java] Insert batch of 10000 records took | 537 | milliseconds
     [java] Insert batch of 10000 records took | 511 | milliseconds
     [java] Insert batch of 10000 records took | 554 | milliseconds
     [java] Insert batch of 10000 records took | 556 | milliseconds
     [java] Insert batch of 10000 records took | 513 | milliseconds
     [java] Insert batch of 10000 records took | 503 | milliseconds
     [java] Insert batch of 10000 records took | 468 | milliseconds
     [java] Insert batch of 10000 records took | 539 | milliseconds
     [java] Insert batch of 10000 records took | 581 | milliseconds
     [java] Insert batch of 10000 records took | 569 | milliseconds
     [java] Insert batch of 10000 records took | 541 | milliseconds
     [java] Insert batch of 10000 records took | 544 | milliseconds
     [java] Insert batch of 10000 records took | 626 | milliseconds
     [java] Insert batch of 10000 records took | 613 | milliseconds
     [java] Insert batch of 10000 records took | 662 | milliseconds
     [java] Insert batch of 10000 records took | 594 | milliseconds
     [java] Insert batch of 10000 records took | 644 | milliseconds
     [java] Insert batch of 10000 records took | 637 | milliseconds
     [java] Insert batch of 10000 records took | 675 | milliseconds
     [java] Insert batch of 10000 records took | 658 | milliseconds
     [java] Insert batch of 10000 records took | 608 | milliseconds
     [java] Insert batch of 10000 records took | 624 | milliseconds
     [java] Insert batch of 10000 records took | 538 | milliseconds
     [java] Insert batch of 10000 records took | 511 | milliseconds
     [java] Insert batch of 10000 records took | 484 | milliseconds
     [java] Insert batch of 10000 records took | 440 | milliseconds
     [java] Insert batch of 10000 records took | 462 | milliseconds
     [java] Insert batch of 10000 records took | 476 | milliseconds
     [java] Elapsed time in seconds 8.502000

BUILD SUCCESSFUL
Total time: 9 seconds


Tuesday, 14 May 2013

Naming Members in vFabric SQLFire

I find it useful to give a member a meaningful  name. In SQLFire you could simply give each member a name by adding a property "name" as follows to the sqlfire.properties file for the member.

sqlfire.properties

# sqlfire.properties for data store or accessor member
license-serial-number=XXXXXXXXXXX
name=server1


Note: The same can be done with GemFire as well.

Then when the system is up the ID for each system member includes the given name as shown below.

  
sqlf> select substr(id, 1 , 35) as "Member" from sys.members;
Member                             
-----------------------------------
172.16.62.1(server2:38971)<v2>:4265
172.16.62.1(server1:38970)<v1>:1660
127.0.0.1(38744):29535             

3 rows selected

Wednesday, 1 May 2013

Explain Plan in vFabric SQLFire improved

With the recently released vFabric SQLFire 11 version the query execution plan is much easier to read then previously. An example below.

  
[Wed May 01 11:32:11 papicella@:~/sqlfire/vFabric_SQLFire_11_b40332/pasdemos/sqlfire ] $ sqlf
sqlf version 10.4
sqlf> connect peer 'bind-address=localhost;mcast-port=12333;host-data=false' as peerClient;
sqlf> explain select * from emp where deptno = 20;
MEMBER_PLAN                                                                                                                     
--------------------------------------------------------------------------------------------------------------------------------
ORIGINATOR 192.168.14.167(73118)<v6>:61492 BEGIN TIME 2013-05-01 11:32:39.735 END TIME 2013-05-01 11:32:39.777
DISTRIBUTION to &
Slowest Member Plan:
member 192.168.14.167(72048)<v1>:42223 begin_execution 2013-05-01 11:32:39.74 end_execution 2013-05-01 11:&
Fastest Member Plan:
member 192.168.14.167(72048)<v1>:42223 begin_execution 2013-05-01 11:32:39.74 end_execution 2013-05-01 11:&

3 rows selected
sqlf> select STMT_ID, STMT_TEXT from SYS.STATEMENTPLANS;
STMT_ID                             |STMT_TEXT                                                                                                                       
---------------------------------------------------------------------------------------------------------------------------------------------------------------------
00000001-ffff-ffff-ffff-000400000016| select * from emp where deptno = <?>                                                                                           

1 row selected
sqlf> explain '00000001-ffff-ffff-ffff-000400000016';
stmt_id 00000001-ffff-ffff-ffff-000400000016 SQL_stmt select * from emp where deptno = <?> begin_execution 2013-05-01 11:32:39.735 end_execution 2013-05-01 11:32:39.777
QUERY-SCATTER  execute_time 0.0 ms
  QUERY-SEND 
    RESULT-RECEIVE 
      SEQUENTIAL-ITERATION (0.38%) execute_time 0.136 ms returned_rows 5 no_opens 1
        RESULT-HOLDER  returned_rows 5 no_opens 1
          DISTRIBUTION-END (99.61%) execute_time 35.073 ms returned_rows 5
member 192.168.14.167(72048)<v1>:42223 begin_execution 2013-05-01 11:32:39.74 end_execution 2013-05-01 11:32:39.774
QUERY-RECEIVE 
  RESULT-SEND 
    RESULT-HOLDER  returned_rows 5 no_opens 1
      ROWIDSCAN (1.71%) execute_time 0.148 ms returned_rows 5 no_opens 1 node_details EMP : 
        CONSTRAINTSCAN (98.28%) execute_time 8.482 ms returned_rows 5 no_opens 1 scan_qualifiers None scanned_object APP.6__EMP__DEPTNO:base-table:APP.EMP scan_type  node_details WHERE : ((DEPTNO = CONSTANT:20) and true) 

Thursday, 11 April 2013

ADO .NET c# Connection Pooling with vFabric SQLFire

Whether it's accessing a database using JAVA or in this case c# I always want to use a connection pool and in this example I show and simple way to do this with a c# ADO .NET client accessing SQLFire.

1. Add a reference to your Visual Studio project in the VMware.Data.SQLFire.dll. This DLL is installed in the  

vFabric_SQLFire_11_bNNNNN\adonet\lib directory.


2. Reference the driver namespace in each source file where you want to use SQLFire components. For example, include this directive with all other references required in your application:

using VMware.Data.SQLFire;

3.  Create a c# console application as follows
  
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VMware.Data.SQLFire;
using System.Data;

namespace SQLFireDemo
{
    class QueryDemo
    {
        private string sqlfHost = "192.168.1.4";
        private int sqlfPort = 1527;

        public QueryDemo()
        {
        }

        private string GetConnectionString()
        {
            return string.Format(@"server={0}:{1}", sqlfHost, sqlfPort);
        }

        public void run()
        {
            using (SQLFClientConnection conn = new SQLFClientConnection(GetConnectionString()))
            {
                conn.Open();
                SQLFCommand command = new SQLFCommand
                      (string.Format("SELECT * FROM dept"), conn);
                SQLFDataReader reader = command.ExecuteReader();

                try
                {
                    StringBuilder row = new StringBuilder();
                    while (reader.Read())
                    {
                        Console.WriteLine(string.Format("Dept[deptno={0}, dname={1}]",
                                          reader.GetString(0),
                                          reader.GetString(1)));
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                }
                finally
                {
                    reader.Close();
                }
            }

        }

        static void Main(string[] args)
        {
            QueryDemo test = new QueryDemo();
            test.run();
        }
    }
}

Output as follows

Dept[deptno=10, dname=ACCOUNTING]
Dept[deptno=20, dname=RESEARCH]
Dept[deptno=30, dname=SALES]
Dept[deptno=40, dname=OPERATIONS]
Dept[deptno=50, dname=MARKETING]
Dept[deptno=60, dname=DEV]
Dept[deptno=70, dname=SUPPORT]
Press any key to continue . . .

Wednesday, 19 December 2012

Spring Data JPA Repository with vFabric SQLFire

Spring JPA is part of the umbrella Spring Data project that makes it easy to easily implement JPA based repositories. In this example below we use Hibernate 4.1 as the JPA implementation along with with our Spring Data JPA repository to get up and running as quickly as possible. Finally in this example we use the SQLFire Hibernate Dialect given we are connecting to a SQLFire distributed database system. More info on the SQLFire Hibernate Dialect can be found here.

The project I have created in STS is as follows.


In no particular order lets show how we defined this project. Like most other blog entries this is based on the classic DEPT/EMP RDBMS tables. Before we start the maven STS project has the following defined. This will ensure you can use this code without compilation errors as all libraries required are defined with maven dependencies.

pom.xml
  
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>spring-hibernate-jpa-sqlfire</groupId>
  <artifactId>spring-hibernate-jpa-sqlfire</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <name>spring-hibernate-jpa-sqlfire</name>
   <properties>
       <spring.version>3.1.2.RELEASE</spring.version>
   </properties>
  <dependencies>
   <dependency>
    <groupId>org.hibernate</groupId>
    <artifactId>hibernate-entitymanager</artifactId>
    <version>4.1.6.Final</version>
   </dependency>
   <dependency>
    <groupId>org.hibernate.javax.persistence</groupId>
    <artifactId>hibernate-jpa-2.0-api</artifactId>
    <version>1.0.0.Final</version>
   </dependency>   
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-core</artifactId>
    <version>${spring.version}</version>
   </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-aop</artifactId>
            <version>${spring.version}</version>
        </dependency>
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-tx</artifactId>
    <version>${spring.version}</version>
   </dependency>
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-orm</artifactId>
    <version>${spring.version}</version>
   </dependency>
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-jdbc</artifactId>
    <version>${spring.version}</version>
   </dependency>
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-context</artifactId>
    <version>${spring.version}</version>
   </dependency>
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-test</artifactId>
    <version>${spring.version}</version>
   </dependency>
   <dependency>
    <groupId>org.springframework</groupId>
    <artifactId>spring-jpa</artifactId>
    <version>2.0.8</version>
   </dependency>
   <dependency>
    <groupId>junit</groupId>
    <artifactId>junit</artifactId>
    <version>4.8.2</version>
   </dependency>
     <dependency>
     <groupId>org.springframework.data</groupId>
     <artifactId>spring-data-jpa</artifactId>
     <version>1.2.0.RELEASE</version>
    </dependency>
    <dependency>
      <groupId>com.vmware</groupId>
      <artifactId>sqlf-client</artifactId>
      <version>1.0.3</version>
      <scope>system</scope>
      <systemPath>/Users/papicella/sqlfire/vFabric_SQLFire_103/lib/sqlfireclient.jar</systemPath>
    </dependency>
    <dependency>
      <groupId>com.vmware</groupId>
      <artifactId>sqlf-dialect</artifactId>
      <version>1.0.3</version>
      <scope>system</scope>
      <systemPath>/Users/papicella/sqlfire/vFabric_SQLFire_103/lib/sqlfHibernateDialect.jar</systemPath>
    </dependency>
  </dependencies>   
    
</project>
1. Create a persistence.xml file in META-INF as shown below.

persistence.xml
  
<?xml version="1.0" encoding="UTF-8"?>
<persistence xmlns="http://java.sun.com/xml/ns/persistence"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_1_0.xsd"
             version="1.0">

 <persistence-unit name="application" transaction-type="RESOURCE_LOCAL">
     <mapping-file>META-INF/persistence-query.xml</mapping-file>
        <class>pas.au.spring.hibernate.sqlfire.model.Dept</class>
        <class>pas.au.spring.hibernate.sqlfire.model.Emp</class>
  <exclude-unlisted-classes>true</exclude-unlisted-classes>
 </persistence-unit>

</persistence> 

2. Optionally created named queries in it's own XML. I prefer to do this then add annotations as this gives you the ability to tweak the queries without altering the code.

persistence-query.xml
  
<?xml version="1.0" encoding="UTF-8" ?>
 
<entity-mappings xmlns="http://java.sun.com/xml/ns/persistence/orm"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://java.sun.com/xml/ns/persistence/orm http://java.sun.com/xml/ns/persistence/orm_1_0.xsd"
        version="1.0">

    <named-query name="Dept.findByDeptnoNamedQuery">
        <query>
            from Dept
            where deptno = ?1
        </query>
    </named-query>
        
</entity-mappings>

3. Create the Dept and Emp domain model classes as shown below. These are referenced in persistence.xml above.

Dept.java
  
package pas.au.spring.hibernate.sqlfire.model;

import java.util.List;

import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.OneToMany;

@Entity
public class Dept 
{
 @Id
 @GeneratedValue (strategy=GenerationType.IDENTITY)
 @Column(name="deptno")
 private Long deptno;
 
 @Column(length = 20, unique = true)
 private String dname;
 
 private String loc;

    @OneToMany(mappedBy = "dept", fetch = FetchType.EAGER)
    private List<Emp> empList;

 public Dept()
 { 
 }
 
 public Dept(Long deptno, String dname, String loc) 
 {
  super();
  this.deptno = deptno;
  this.dname = dname;
  this.loc = loc;
 }

 public Long getDeptno() {
  return deptno;
 }

 public void setDeptno(Long deptno) {
  this.deptno = deptno;
 }

 public String getDname() {
  return dname;
 }

 public void setDname(String dname) {
  this.dname = dname;
 }

 public String getLoc() {
  return loc;
 }

 public void setLoc(String loc) {
  this.loc = loc;
 }

 
    public List<Emp> getEmpList() {
  return empList;
 }

 public void setEmpList(List<Emp> empList) {
  this.empList = empList;
 }

 public Emp addEmp(Emp emp) {
        getEmpList().add(emp);
        emp.setDept(this);
        return emp;
    }

    public Emp removeEmp(Emp emp) {
        getEmpList().remove(emp);
        emp.setDept(null);
        return emp;
    }

 @Override
 public String toString() {
  return "Dept [deptno=" + deptno + ", dname=" + dname + ", loc=" + loc
    + "]";
 }
 
} 
Emp.java
  
package pas.au.spring.hibernate.sqlfire.model;

import java.sql.Timestamp;

import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;

@Entity
public class Emp 
{
 @Id
 @GeneratedValue (strategy=GenerationType.IDENTITY)
 @Column(name="empno")
 private Long empno;
 
 @Column(length = 20)
 private String ename;

 private Timestamp hiredate;
    private String job;
    private Long mgr;
    private Double sal;
    @ManyToOne
    @JoinColumn(name = "DEPTNO")
    private Dept dept;
    
    public Emp()
    {
    }

 public Emp(Long empno, String ename, String job, Long mgr, Double sal, Timestamp hiredate, Dept dept) {
  super();
  this.empno = empno;
  this.ename = ename;
  this.job = job;
  this.mgr = mgr;
  this.sal = sal;
  this.hiredate = hiredate;
  this.dept = dept;
 }

 public Long getEmpno() {
  return empno;
 }

 public void setEmpno(Long empno) {
  this.empno = empno;
 }

 public String getEname() {
  return ename;
 }

 public void setEname(String ename) {
  this.ename = ename;
 }

 public String getJob() {
  return job;
 }

 public void setJob(String job) {
  this.job = job;
 }

 public Long getMgr() {
  return mgr;
 }

 public void setMgr(Long mgr) {
  this.mgr = mgr;
 }

 public Double getSal() {
  return sal;
 }

 public void setSal(Double sal) {
  this.sal = sal;
 }

 public Dept getDept() {
  return dept;
 }

 public void setDept(Dept dept) {
  this.dept = dept;
 }

 public Timestamp getHiredate() {
  return hiredate;
 }

 public void setHiredate(Timestamp hiredate) {
  this.hiredate = hiredate;
 }

 @Override
 public String toString() {
  return "Emp [empno=" + empno + ", ename=" + ename + ", hiredate="
    + hiredate + ", job=" + job + ", mgr=" + mgr + ", sal=" + sal
    + ", dept=" + dept + "]";
 }
 
} 
4. Create a spring XML file in the META-INF/spring directory named "applicationContext-persistence.xml"

applicationContext-persistence.xml
  
<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"
 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 xmlns:tx="http://www.springframework.org/schema/tx"
 xmlns:repository="http://www.springframework.org/schema/data/repository"
 xmlns:jpa="http://www.springframework.org/schema/data/jpa"
 xsi:schemaLocation="http://www.springframework.org/schema/data/repository http://www.springframework.org/schema/data/repository/spring-repository-1.4.xsd
  http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
  http://www.springframework.org/schema/data/jpa http://www.springframework.org/schema/data/jpa/spring-jpa-1.2.xsd
  http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd">
 
    <bean id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
        <property name="jpaDialect">
            <bean class="org.springframework.orm.jpa.vendor.HibernateJpaDialect" />
        </property>
        <property name="jpaVendorAdapter">
            <bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter" />
        </property>
        <property name="persistenceUnitName" value="application" />
        <property name="persistenceUnitManager">
            <bean class="org.springframework.orm.jpa.persistenceunit.DefaultPersistenceUnitManager" />
        </property>
        <property name="jpaProperties">
            <props>
                <prop key="hibernate.dialect">com.vmware.sqlfire.hibernate.SQLFireDialect</prop>
                <prop key="hibernate.show_sql">true</prop>
    <prop key="hibernate.format_sql">true</prop>
                <prop key="hibernate.connection.driver_class">com.vmware.sqlfire.jdbc.ClientDriver</prop>
                <prop key="hibernate.connection.url">jdbc:sqlfire://localhost:1527</prop>
                <prop key="hibernate.connection.username">APP</prop>
                <prop key="hibernate.connection.password">APP</prop>
            </props>
        </property>
        <property name="packagesToScan" value="pas.au.spring.hibernate.sqlfire.repository" />
    </bean>
 
    <bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
        <property name="entityManagerFactory" ref="entityManagerFactory" />
    </bean>
 
    <tx:annotation-driven/>

 
 <!-- CRUD JPA Repositories Here loaded by default  -->
 <jpa:repositories base-package="pas.au.spring.hibernate.sqlfire.repository" />
 
</beans>

5. Create a JPA CRUD repository as shown below. This will automatically be picked up by spring JPA repository component scanner as defined in "applicationContext-persistence.xml" above.

JPADeptRepository.java
  
package pas.au.spring.hibernate.sqlfire.repository;

import java.util.List;

import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.CrudRepository;

import pas.au.spring.hibernate.sqlfire.model.Dept;

/*
 * Define a repository with all CRUD methods written for you
 */
public interface JPADeptRepository extends CrudRepository<Dept, Long>
{
 /*
  * Method which will invoke the JPQL required for a find by the attribute name
  */
 public Dept findByDeptno (Long deptno);
 
 /* 
  * Shows how to access  a named query using a method name to identify it
  * The method name must match the defined query name
  * 
  */
 public Dept findByDeptnoNamedQuery (Long deptno);

 /*
  * Method that allows us to supply a JPQL query to execute for the method
  */
 @Query("SELECT d FROM Dept d WHERE d.dname LIKE '%'")
    public List<Dept> findDeptsWithCustomJPQLQuery();

 /*
  * Method that allows us to supply a JPQL query to execute for the method with a parameter
  */
 @Query("SELECT d FROM Dept d WHERE d.loc = ?1")
    public Dept findDeptWithCustomJPQLQueryWithParam(String loc);
 
 @Query("SELECT d FROM Dept d WHERE d.deptno = ?1")
 public Dept findEmpsInDeptno (Long deptno);
 
 @Query(value = "SELECT * FROM DEPT where deptno = ?1", nativeQuery = true)
 public Dept findDeptByNativeQuery(Long deptno);
 
 @Query(value = "SELECT d, e FROM Dept d, Emp e where e.dept = d and d.deptno = ?1")
 public List<Object[]> findWithJoin(Long deptno);

} 

6. Finally create a test class to verify the repository. It's assumed you have SQLFire running and the DEPT/EMP table already exist. You can use this previous blog entry to get that setup.

http://theblasfrompas.blogspot.com.au/2011/12/sqlfire-in-few-minutes.html

JPAReposDeptTest.java
  
package pas.au.spring.hibernate.sqlfire.test.repository;

import static org.junit.Assert.*;
import java.util.List;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import pas.au.spring.hibernate.sqlfire.repository.JPADeptRepository;
import pas.au.spring.hibernate.sqlfire.model.Dept;
import pas.au.spring.hibernate.sqlfire.model.Emp;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration("classpath:META-INF/spring/applicationContext-persistence.xml")
public class JPAReposDeptTest 
{
 @Autowired
 JPADeptRepository deptRespository;
 
 @Test
 public void invokeFindAll() 
 {
  List<Dept> deps = (List<Dept>) deptRespository.findAll();
  
  System.out.println("deps size = " + deps.size());
  assertNotNull(deps);
 }

 @Test
 public void invokeFindOne() 
 {
  Dept dept = deptRespository.findOne(new Long(20));
  
  System.out.println(dept);
  assertNotNull(dept);
  assertEquals(new Long(20), dept.getDeptno());
 }
 
 @Test
 public void invokeFindByDeptno() 
 {
  Dept dept = deptRespository.findByDeptno(new Long(10));
  
  System.out.println(dept);
  assertNotNull(dept);
  assertEquals(new Long(10), dept.getDeptno());
 }

 @Test
 public void invokeNamedQuery() 
 {
  Dept dept = deptRespository.findByDeptnoNamedQuery(new Long(30));
  
  System.out.println(dept);
  assertNotNull(dept);
  assertEquals(new Long(30), dept.getDeptno());
 }

 @Test
 public void invokefindDeptsWithCustomJPQLQuery() 
 {
  List<Dept> deps = (List<Dept>) deptRespository.findDeptsWithCustomJPQLQuery();
  
  System.out.println("deps size = " + deps.size());
  assertNotNull(deps);
 }

 @Test
 public void invokefindDeptsWithCustomJPQLQueryWithParam() 
 {
  Dept dept = deptRespository.findDeptWithCustomJPQLQueryWithParam("CHICAGO");
  
  System.out.println(dept);
  assertNotNull(dept);
  assertEquals(new Long(30), dept.getDeptno());
 }

 @Test
 public void invokefindEmpInDeptno() 
 {
  Dept dept = deptRespository.findEmpsInDeptno(new Long(20));
  
  System.out.println(dept);
  assertNotNull(dept);
  
  List<Emp> emps = dept.getEmpList();
  for (Emp e: emps)
  {
   System.out.println(e);
  }
  
  assertEquals(5, emps.size());
 }
 
 @Test
 public void invokeFindDeptByNativeQuery()
 {
  Dept dept = deptRespository.findDeptByNativeQuery(new Long(40));
  
  System.out.println(dept);
  assertNotNull(dept);
  assertEquals(new Long(40), dept.getDeptno());  
 }
 
 @Test
 public void invokeFindWithJoin()
 {
  List<Object[]> results = deptRespository.findWithJoin(new Long(10));
  
  System.out.println("Size of results = " + results.size());
  assertEquals(3, results.size());
  
  for (Object[] result: results)
  {
   System.out.println("Dept: " + result[0] + ", Emp: " + result[1]);
  }
  
 }
}

7. Run the test class above.

Verify output as shown below.



Dec 19, 2012 9:11:41 PM org.hibernate.engine.transaction.internal.TransactionFactoryInitiator initiateService
INFO: HHH000268: Transaction strategy: org.hibernate.engine.transaction.internal.jdbc.JdbcTransactionFactory
Dec 19, 2012 9:11:41 PM org.hibernate.hql.internal.ast.ASTQueryTranslatorFactory
INFO: HHH000397: Using ASTQueryTranslatorFactory
Hibernate:
    select
        dept0_.deptno as deptno0_,
        dept0_.dname as dname0_,
        dept0_.loc as loc0_
    from
        Dept dept0_
Hibernate:
    select
        emplist0_.DEPTNO as DEPTNO0_1_,
        emplist0_.empno as empno1_,
        emplist0_.empno as empno1_0_,
        emplist0_.DEPTNO as DEPTNO1_0_,
        emplist0_.ename as ename1_0_,
        emplist0_.hiredate as hiredate1_0_,
        emplist0_.job as job1_0_,
        emplist0_.mgr as mgr1_0_,
        emplist0_.sal as sal1_0_
    from
        Emp emplist0_
    where
        emplist0_.DEPTNO=?

.......
Size of results = 3
Dept: Dept [deptno=10, dname=ACCOUNTING, loc=NEW YORK], Emp: Emp [empno=7934, ename=MILLER, hiredate=1982-01-23 00:00:00.0, job=CLERK, mgr=7782, sal=1300.0, dept=Dept [deptno=10, dname=ACCOUNTING, loc=NEW YORK]]
Dept: Dept [deptno=10, dname=ACCOUNTING, loc=NEW YORK], Emp: Emp [empno=7782, ename=CLARK, hiredate=1981-06-09 00:00:00.0, job=MANAGER, mgr=7839, sal=2450.0, dept=Dept [deptno=10, dname=ACCOUNTING, loc=NEW YORK]]
Dept: Dept [deptno=10, dname=ACCOUNTING, loc=NEW YORK], Emp: Emp [empno=7839, ename=KING, hiredate=1981-11-17 00:00:00.0, job=PRESIDENT, mgr=null, sal=5000.0, dept=Dept [deptno=10, dname=ACCOUNTING, loc=NEW YORK]]


More Information on Spring data JPA and SQLFire Hibernate Dialect can be can be found here.

Spring Data JPA
http://www.springsource.org/spring-data/jpa

vFabric SQLFire Hibernate Dialect
http://communities.vmware.com/docs/DOC-20294


Friday, 14 September 2012

SQLFire - Using Procedure Parameters with DYNAMIC RESULT SETS

In this example below we use DYNAMIC RESULT SETS with stored procedures. When you configure a procedure using the CREATE PROCEDURE statement, SQLFire assembles the method parameters and passes them to the procedure implementation class using Java reflection

The demo below is using the classic DEPT/EMP schema data. If you want to follow it setup SQLFire with that data as shown in this previous blog entry.

http://theblasfrompas.blogspot.com/2011/12/sqlfire-in-few-minutes.html

1. Create a Java Stored Procedure class with a single method as follows.
package pas.au.vmware.sqlfire.procs;

import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.PreparedStatement;

public class DemoProc
{
  public static void queryEmpsinDepartments(String deptno, ResultSet[] resultSet1) throws SQLException
  {
    String sql = "select empno, ename, deptno from emp where deptno = ?";
    
    Connection conn = null;
    PreparedStatement pstmt = null;
    ResultSet rset = null;
    
    try
    {
      conn = DriverManager.getConnection("jdbc:default:connection");
      pstmt = conn.prepareStatement(sql);
      pstmt.setInt(1, Integer.parseInt(deptno));
      resultSet1[0] = pstmt.executeQuery();
      
    }
    catch (SQLException se)
    {
      throw new RuntimeException("Error in stored procedure queryEmpsinDepartments - ", se);
    }
  }
}
2. Add the JAR file to the classpath for your SQLFire distributed system prior to starting the system. Here we just use the CLASSPATH variable but you could also load the JAR file into the system itself.

export CUR_DIR=`pwd`
export CLASSPATH=$CUR_DIR/lib/procs.jar

3. Create a stored procedure as follows.
CREATE PROCEDURE queryEmpsinDepartments (IN deptno VARCHAR(2))
LANGUAGE JAVA 
PARAMETER STYLE JAVA 
READS SQL DATA 
DYNAMIC RESULT SETS 1 
EXTERNAL NAME 'pas.au.vmware.sqlfire.procs.DemoProc.queryEmpsinDepartments';
4. Invoke from a client as shown below.
package clientaccess;

import java.sql.*;

public class TestQueryEmpsinDepartments
{
  public TestQueryEmpsinDepartments()
  {
  }


  public static Connection getConnection() throws SQLException
  {

    String thinConn = "jdbc:sqlfire://localhost:1527/";
    Connection conn = DriverManager.getConnection(thinConn);
    conn.setAutoCommit(false);
    return conn;
  }
  
  public void run () throws SQLException
  {
    String sql = "call queryEmpsinDepartments(?)";
    
    Connection conn = null;
    PreparedStatement pstmt = null;
    ResultSet rset = null;
    
    try
    {
      conn = getConnection();
      pstmt = conn.prepareCall(sql);
      pstmt.setString(1, "10");
      pstmt.execute();
      
      rset = pstmt.getResultSet();
      
      System.out.println("** Employees in department with id 10 **");
      
      while (rset.next())
      {
        System.out.println(String.format("Emp[%s, %s, %s]", rset.getInt(1), rset.getString(2), rset.getInt(3)));
      }
    }
    finally
    {
      if (rset != null)
      {
        rset.close();
      }
      
      if (pstmt != null)
      {
        pstmt.close();
      }
      
      if (conn != null)
      {
        conn.close();
      }
    }
    
  }
  
  public static void main(String[] args) throws SQLException
  {
    TestQueryEmpsinDepartments test = new TestQueryEmpsinDepartments();
    System.out.println("Started at " + new java.util.Date());
    test.run();
    System.out.println("Finished at " + new java.util.Date());
  }
} 

Output

Started at Fri Sep 14 10:59:20 EST 2012
** Employees in department with id 10 **
Emp[7934, MILLER, 10]
Emp[7782, CLARK, 10]
Emp[7839, KING, 10]
Finished at Fri Sep 14 10:59:20 EST 2012

For more information see the link below.

http://pubs.vmware.com/vfabric5/index.jsp?topic=/com.vmware.vfabric.sqlfire.1.0/developers_guide/topics/server-side/dap-impl-parameters.html

Thursday, 21 June 2012

Loading existing RDBMS data into SQLFire

A common scenario or question I get is how do I load data into SQLFire from an existing RDBMS. If you have a simple schema and you wish to load that into SQLFire you can use Apache DDLUtils as shown below. In this example we take the classic DEPT/EMP schema from MYSQL and place it into SQLFire.

1. Create a build.xml as follows. This example is taken from $SQLFIRE_HOME/lib/ddlutils/example directory. It's been edited to point to a MYSQL database for the source schema and a target SQLFire database as the target schema.
  
<?xml version="1.0"?>

<project name="test" default="writeDDLToXML" basedir=".">

<path id="runtime-classpath">
  <fileset dir="./lib">
    <include name="**/*.jar"/>
    <include name="**/*.zip"/>
  </fileset>
</path>

 <!-- This will output all the object in the database 'test' and output the schema definition
   to a file 'db-schema1.xml'. This will also extract the data and output this to 'data.xml'
  -->
<target name="writeDDLToXML" description="Dumps the database structure">
  <taskdef name="databaseToDdl"
           classname="org.apache.ddlutils.task.DatabaseToDdlTask">
    <classpath refid="runtime-classpath"/>

  </taskdef>
  <databaseToDdl verbosity="debug">
  <!--
    <database url="jdbc:sqlfire://localhost:1527"
              driverClassName="com.vmware.sqlfire.jdbc.ClientDriver"
              username="app"
              password="app"/>
  -->

    <database url="jdbc:mysql://localhost/scottdb"
              driverClassName="com.mysql.jdbc.Driver"
              username="scott"
              password="tiger"/>

    <writeSchemaToFile outputFile="db-schema1.xml" failonerror="false" />

    <!-- Comment line below if the source DB is too big -->
    <writeDataToFile outputFile="data.xml"/>
  </databaseToDdl>

</target>


 <!-- This will create the tables, etc in SQLFire.
      If your table names or column names contain embedded spaces, set
      usedelimitedsqlidentifiers to true.
  -->
<target name="createDBFromXML" description="Create the DB tables ..">
  <taskdef classname="org.apache.ddlutils.task.DdlToDatabaseTask"
          name="ddlToDatabase"
          classpathref="runtime-classpath"/>

   <ddlToDatabase usedelimitedsqlidentifiers="false">
     <database driverclassname="com.vmware.sqlfire.jdbc.ClientDriver"
             url="jdbc:sqlfire://localhost:1527"
             username="app"
             password="app"/>
   <fileset dir=".">
     <include name="db-schema1.xml"/>
   </fileset>

   <!--   <createdatabase failonerror="false"/>  -->

   <writeschematodatabase alterdatabase="true"
                          failonerror="false"/>
   </ddlToDatabase>
</target>


 <!-- Extract DDL for a schema and write this out as a 'SQL' script file (db-schema1.sql).
      EDIT THIS FILE TO CHANGE THE TABLES TO BE CUSTOM PARTITIONED, REPLICATED, PERSISTENT, ETC
      Then, execute the SQL script using 'IJ' or your favorite tool (like SQuirrel)
 -->
<target name="writeDDLToSQL" description="Dumps the database structure">
  <taskdef name="databaseToDdl"
           classname="org.apache.ddlutils.task.DatabaseToDdlTask">
    <classpath refid="runtime-classpath"/>

  </taskdef>
  <databaseToDdl verbosity="debug">
    <database url="jdbc:sqlfire://localhost:1527"
              driverClassName="com.vmware.sqlfire.jdbc.ClientDriver"
              username="app"
              password="app"/>

    <writeSchemaSqlToFile outputFile="db-schema1.sql" dodrops="true" failonerror="false" alterdatabase="false"/>
  </databaseToDdl>
</target>


 <!-- Imports data rows into a database.
      If your table names or column names contain embedded spaces, set
      usedelimitedsqlidentifiers to true.
  -->
<target name="ImportDataToDB" description="Import the data ..">
  <taskdef classname="org.apache.ddlutils.task.DdlToDatabaseTask"
          name="ddlToDatabase"
          classpathref="runtime-classpath"/>

   <ddlToDatabase usedelimitedsqlidentifiers="false">
    <database url="jdbc:sqlfire://localhost:1527"
              driverClassName="com.vmware.sqlfire.jdbc.ClientDriver"
              username="app"
              password="app"/>
   <fileset dir=".">
     <include name="db-schema1.xml"/>
   </fileset>

   <writedatatodatabase datafile="data.xml"
                        usebatchmode="true"
                        batchsize="1000"/>
 </ddlToDatabase>
</target>

</project>

2. Ensure you have Apache ant in your path
3. Ensure you have placed the JDBC drivers for SQLFire and MYSQL in the path which will be picked up by ant. In this example ANT is using a "lib" sub directory to include all JARS in that directory at runtime. This "lib" directory would also include the Apache DDLUtils jar files as well.

mysql-connector-java-5.1.17-bin.jar
sqlfireclient.jar

Here is a full directory listing on the "lib" directory JAR files I was using.

[Thu Jun 21 14:29:01 papicella@:~/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/lib ] $ d
total 38848
-rw-r--r--   1 papicella  staff    787920  4 Jul  2011 mysql-connector-java-5.1.17-bin.jar
-rw-r--r--   1 papicella  staff    474464 20 Jun 14:32 wstx-asl-3.0.2.jar
-rw-r--r--   1 papicella  staff     26514 20 Jun 14:32 stax-api-1.0.1.jar
-rw-r--r--   1 papicella  staff    352668 20 Jun 14:32 log4j-1.2.8.jar
-rw-r--r--   1 papicella  staff    486522 20 Jun 14:32 dom4j-1.4.jar
-rw-r--r--   1 papicella  staff     42492 20 Jun 14:32 commons-pool-1.2.jar
-rw-r--r--   1 papicella  staff     38015 20 Jun 14:32 commons-logging-1.0.4.jar
-rw-r--r--   1 papicella  staff    207723 20 Jun 14:32 commons-lang-2.1.jar
-rw-r--r--   1 papicella  staff    139966 20 Jun 14:32 commons-digester-1.7.jar
-rw-r--r--   1 papicella  staff    107631 20 Jun 14:32 commons-dbcp-1.2.1.jar
-rw-r--r--   1 papicella  staff    559366 20 Jun 14:32 commons-collections-3.1.jar
-rw-r--r--   1 papicella  staff     46725 20 Jun 14:32 commons-codec-1.3.jar
-rw-r--r--   1 papicella  staff    188671 20 Jun 14:32 commons-beanutils-1.7.0.jar
-rw-r--r--   1 papicella  staff    447398 20 Jun 14:34 DdlUtils-1.0.jar
-rw-r--r--   1 papicella  staff    822779 20 Jun 14:37 sqlfireclient.jar
-rw-r--r--   1 papicella  staff  15125561 20 Jun 14:37 sqlfire.jar
drwxr-xr-x  18 papicella  staff       612 21 Jun 14:08 ./
drwxr-xr-x   7 papicella  staff       238 21 Jun 14:08 ../


** At this point you can begin the process as shown below **

4. Create schema / data file in XML using the ant task as follows

[Thu Jun 21 08:36:10 papicella@:~/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils ] $ ant writeDDLToXML
Buildfile: /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/build.xml

writeDDLToXML:
[databaseToDdl] Borrowed connection org.apache.commons.dbcp.PoolableConnection@bdee400 from data source
[databaseToDdl] Returning connection org.apache.commons.dbcp.PoolableConnection@bdee400 to data source.
[databaseToDdl] Remaining connections: None
[databaseToDdl] Written schema to /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/db-schema1.xml
[databaseToDdl] Borrowed connection org.apache.commons.dbcp.PoolableConnection@bdee400 from data source
[databaseToDdl] Returning connection org.apache.commons.dbcp.PoolableConnection@bdee400 to data source.
[databaseToDdl] Remaining connections: None
[databaseToDdl] Borrowed connection org.apache.commons.dbcp.PoolableConnection@bdee400 from data source
[databaseToDdl] Returning connection org.apache.commons.dbcp.PoolableConnection@bdee400 to data source.
[databaseToDdl] Remaining connections: None
[databaseToDdl] Written data XML to file/Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/data.xml

BUILD SUCCESSFUL
Total time: 1 second

5. If your happy to use only replicated tables then you can import the schema definition into
SQLFire as follows. Every table created by default is "replicate" in this scenario.

[Thu Jun 21 08:36:37 papicella@:~/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils ] $ ant createDBFromXML
Buildfile: /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/build.xml

createDBFromXML:
[ddlToDatabase] Read schema file /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/db-schema1.xml
[ddlToDatabase] Table DEPT needs to be added
[ddlToDatabase] Table EMP needs to be added
[ddlToDatabase] Foreign key Foreign key [name=EMP_FK; foreign table=DEPT; 1 references] needs to be added to table EMP
[ddlToDatabase] Executed 3 SQL command(s) with 0 error(s)
[ddlToDatabase] Written schema to database

BUILD SUCCESSFUL
Total time: 0 seconds

6. This next step will take the XML created in step #1 and create a schema file in SQL. This is great because you can now add the SQLFire SQL to use partioned, disk stores etc. Without persisteant disk stores the data will disappear once the system goes down.

[Thu Jun 21 08:54:01 papicella@:~/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils ] $ ant writeDDLToSQL
Buildfile: /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/build.xml

writeDDLToSQL:
[databaseToDdl] Borrowed connection org.apache.commons.dbcp.PoolableConnection@2d1e233 from data source
[databaseToDdl] Returning connection org.apache.commons.dbcp.PoolableConnection@2d1e233 to data source.
[databaseToDdl] Remaining connections: None
[databaseToDdl] Written schema SQL to /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/db-schema1.sql

BUILD SUCCESSFUL
Total time: 1 second

7. Finally with a schema created we can simply import the data created from #4 above as follows.

[Thu Jun 21 08:54:43 papicella@:~/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils ] $ ant ImportDataToDB
Buildfile: /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/build.xml

ImportDataToDB:
[ddlToDatabase] Read schema file /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/db-schema1.xml
[ddlToDatabase] Written data from file /Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo/ddlutils/data.xml to database

BUILD SUCCESSFUL
Total time: 0 seconds

8. The last thing left to do is verify we have our schema data within SQLFire as shown below.
  
[Thu Jun 21 08:55:32 papicella@:~/sqlfire/vFabric_SQLFire_1021/pasdemos/ddlutils-demo ] $ sqlf
sqlf version 10.4
sqlf> connect client 'localhost:1527';
sqlf> show tables in app;
TABLE_SCHEM         |TABLE_NAME                    |REMARKS             
------------------------------------------------------------------------
APP                 |DEPT                          |                    
APP                 |EMP                           |                    

2 rows selected
sqlf> select * from dept;
DEPTNO     |DNAME         |LOC          
----------------------------------------
20         |RESEARCH      |DALLAS       
10         |ACCOUNTING    |NEW YORK     
30         |SALES         |CHICAGO      
40         |OPERATIONS    |BOSTON       

4 rows selected 

For more information on DDLUtils with SQLFire see the link below.

http://pubs.vmware.com/vfabric5/index.jsp?topic=/com.vmware.vfabric.sqlfire.1.0/data_management/ddlutils-procedure.html

Friday, 8 June 2012

Modelling SQLFire with a Locator VM in vFabric Application Director

I recently setup a SQLFire distributed system by modelling a blueprint from vFabric Application Director. In this simple setup I created a service for a locator as well as a SQLFire member. The screen shots below show what this looked like.

New Catalog Items

1. Create the locator service as follows

1.1. Define it as follows


1.2. Set properties as follows


1.3. Add INSTALL script as shown below
  
#!/bin/sh
# Import global conf
. $global_conf

set -e

#To support ubuntu java bins
export PATH=$PATH:/usr/java/jre-vmware/bin

mkdir -p $install_path
java -Dsqlfire.installer.directory="$install_path" -jar `basename "$SQLFire102_Installer"`
 
cd $install_path/*SQLFire*

mkdir locator
./bin/sqlf locator start -peer-discovery-address=$locator_ip -peer-discovery-port=$locator_port -client-bind-address=$locator_ip -client-port=$client_port -dir=locator

2. Create a SQLFire server service

2.1. Define it as follows




2.2. Set properties as follows


2.3 Add install script as shown below
  
#!/bin/sh
# Import global conf
. $global_conf

set -e

#To support ubuntu java bins
export PATH=$PATH:/usr/java/jre-vmware/bin

mkdir -p $install_path
java -Dsqlfire.installer.directory="$install_path" -jar `basename "$SQLFire102_Installer"`
 
cd $install_path/*SQLFire*

mkdir server1
./bin/sqlf server start -dir=server1 -client-port=$client_port -client-bind-address=$server_ip -locators=$locator_ip[$locator_port]

Blueprint using the Catalog Items above

The catalog items are simply added to the templates as needed. From the image below we have one locator VM and a cluster of SQLFire server node VM's as well. This enables us to create as many SQLFire member nodes as needed.


From the screen shot below it shows how we determine the locator IP address as well as the SQLFire server node IP address at runtime as we need that to be injected in for us before the lifecycle INSTALL phase is started.


Monday, 21 May 2012

Simple example on how to caputure the query execution plan from SQLFire

The example below shows how we can capure the query plan for SQL running in SQLFire. Although any user can run an explain plan you must connect as a peer to show the query execution plan as the data is stored in the SYS.STATEMENTPLANS table.

1. Determine what indexes we currently have in the "APP" schema.
  
sqlf> show indexes in app;
TABLE_NAME          |COLUMN_NAME         |NON_U&|TYPE|ASC&|CARDINA&|PAGES   
----------------------------------------------------------------------------
DEPT                |DEPTNO              |0     |3   |A   |NULL    |NULL    
EMP                 |EMPNO               |0     |3   |A   |NULL    |NULL    
EMP                 |DEPTNO              |1     |3   |A   |NULL    |NULL    
EMP                 |JOB                 |1     |3   |A   |NULL    |NULL    

4 rows selected

2. Run a query explain for an individual SQL statement and display the execution plan. In this example below we connect as a peer client to perform the opertion BUT we only need to conect as a peer client to actually display the execution plan and we can explain our SQL as a regular client user.
  
sqlf> connect peer 'host-data=false;mcast-port=12333';
sqlf> explain select * from emp where job = 'CLERK';
MEMBER_PLAN                                                                                                                     
--------------------------------------------------------------------------------------------------------------------------------
ORIGINATOR 192-168-1-4.tpgi.com.au(2201)<v4>:11255/52510 BEGIN TIME 2012-05-21 20:02:03.659 END TIME 2012-05-21 20:02:03.694
DI&
Slowest Member Plan: 
member   192-168-1-4.tpgi.com.au(2064)<v0>:39756/52450 begin_execution  2012-05-21 20:02:03.664 end_execu&
Fastest Member Plan: 
member   192-168-1-4.tpgi.com.au(2065)<v1>:55134/52451 begin_execution  2012-05-21 20:02:03.664 end_execu&

3 rows selected
sqlf> select STMT_ID, STMT_TEXT from SYS.STATEMENTPLANS;
STMT_ID                             |STMT_TEXT                                                                                                                       
---------------------------------------------------------------------------------------------------------------------------------------------------------------------
00000001-ffff-ffff-ffff-000300000016| select * from emp where job = 'CLERK'                                                                                          

1 row selected
sqlf> explain '00000001-ffff-ffff-ffff-000300000016';
stmt_id   00000001-ffff-ffff-ffff-000300000016 begin_execution  2012-05-21 20:02:03.659 end_execution  2012-05-21 20:02:03.694
QUERY-SCATTER execute_time 31696000 member_node 192-168-1-4.tpgi.com.au(2064)<v0>:39756/52450,192-168-1-4.tpgi.com.au(2065)<v1>:55134/52451
  QUERY-SEND execute_time 397000 member_node 192-168-1-4.tpgi.com.au(2064)<v0>:39756/52450
    QUERY-SEND execute_time 178000 member_node 192-168-1-4.tpgi.com.au(2065)<v1>:55134/52451
      RESULT-RECEIVE execute_time 150000 member_node 192-168-1-4.tpgi.com.au(2065)<v1>:55134/52451
        RESULT-RECEIVE execute_time 35000 member_node 192-168-1-4.tpgi.com.au(2064)<v0>:39756/52450
          SEQUENTIAL-ITERATION returned_rows 7 no_opens 1 execute_time 218000
            RESULT-HOLDER returned_rows 2 no_opens 1 execute_time 39000 member_node 192-168-1-4.tpgi.com.au(2065)<v1>:55134/52451
              RESULT-HOLDER returned_rows 5 no_opens 1 execute_time 33000 member_node 192-168-1-4.tpgi.com.au(2064)<v0>:39756/52450
                DISTRIBUTION-END returned_rows 7 execute_time 28521000
member   192-168-1-4.tpgi.com.au(2065)<v1>:55134/52451 begin_execution  2012-05-21 20:02:03.664 end_execution  2012-05-21 20:02:03.691
QUERY-RECEIVE execute_time 26501000 member_node 192-168-1-4.tpgi.com.au(2201)<v4>:11255/52510
  RESULT-SEND execute_time 35000 member_node 192-168-1-4.tpgi.com.au(2201)<v4>:11255/52510
    RESULT-HOLDER returned_rows 2 no_opens 1 execute_time 802000
      ROWIDSCAN returned_rows 2 no_opens 1 execute_time 62000
        INDEXSCAN returned_rows 2 no_opens 1 execute_time 6173000 scan_qualifiers None scanned_object EMP_JOB_IDX scan_type 
member   192-168-1-4.tpgi.com.au(2064)<v0>:39756/52450 begin_execution  2012-05-21 20:02:03.664 end_execution  2012-05-21 20:02:03.692
QUERY-RECEIVE execute_time 27574000 member_node 192-168-1-4.tpgi.com.au(2201)<v4>:11255/52510
  RESULT-SEND execute_time 25000 member_node 192-168-1-4.tpgi.com.au(2201)<v4>:11255/52510
    RESULT-HOLDER returned_rows 5 no_opens 1 execute_time 885000
      ROWIDSCAN returned_rows 5 no_opens 1 execute_time 108000
        INDEXSCAN returned_rows 5 no_opens 1 execute_time 6682000 scan_qualifiers None scanned_object EMP_JOB_IDX scan_type 
sqlf>

So the steps are as follows:

1. Run an explain for the SQL using the key word "EXPLAIN".
2. Connect as a peer
3. Show the query plan execution for the statement.

For more information on the query execution plan codes visit the link below.

http://pubs.vmware.com/vfabric5/index.jsp?topic=/com.vmware.vfabric.sqlfire.1.0/manage_guide/explain-codes.html



Friday, 16 March 2012

Using derby style table functions to load data into SQLFire

Loading data into SQLFire can be done various ways including using Spring Batch with CSV files, Apache DDLUtils or direct JDBC connections pulling data into SQLFire. The approach below is yet another way. In this example we load the Oracle HR schema table "departments" into a SQLFire distributed system.

1. Create a java class with code that simple queries the table data and returns it as aJDBC ResultSet.
package vmware.au.se.sqlfire.derby;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

import oracle.jdbc.OracleDriver;

public class DepartmentTable 
{

 public static String SQL = "select * from departments";
 
 public static ResultSet read() throws SQLException
 {
        Connection conn = getConnection();
        Statement  stmt = conn.createStatement(); 
        ResultSet  rset = stmt.executeQuery(SQL);
        
        return rset;
 }
 
 public static Connection getConnection() throws SQLException
 {
     String username = "hr";
     String password = "hr";
     String thinConn = "jdbc:oracle:thin:@172.16.101.70:1521/linux11gr2";
     DriverManager.registerDriver(new OracleDriver());
     Connection conn = DriverManager.getConnection(thinConn,username,password);
     conn.setAutoCommit(false);
     return conn;
 }
 
} 

2. Add the JAR file with the class above as well as the Oracle JDBC jar file to the system CLASSPATH as shown below.

export CUR_DIR=`pwd`
export CLASSPATH=$CUR_DIR/lib/ojdbc6.jar:$CUR_DIR/lib/derby-funct.jar

Note: This ensures when we start our SQLFire nodes they will have the classes avaiable on the classpath

3. Start the SQLFire servers as shown below.

> sqlf server start -server-groups=MYGROUP -locators=localhost[41111] -client-bind-address=localhost -client-port=1528 -dir=server1 -classpath=$CLASSPATH &

> sqlf server start -server-groups=MYGROUP -locators=localhost[41111] -client-bind-address=localhost -client-port=1529 -dir=server2 -classpath=$CLASSPATH &

4. Log into the distributed system using the CLI and run the SQL as follows to create the table in SQLFire which will store the same dataset from the Oracle "departments" table.
create diskstore STORE1;
 
call sys.set_eviction_heap_percentage_sg (85, 'MYGROUP');

drop table departments;

create table departments
(department_id int NOT NULL CONSTRAINT department_id_PK PRIMARY KEY,
 department_name varchar(40),
 manager_id int,
 location_id int)
partition by column (department_id)
SERVER GROUPS (MYGROUP)
persistent 'STORE1'
REDUNDANCY 1;

5. Log into the distributed system using the CLI and run the SQL below to create a function
CREATE FUNCTION externalDepartments
()
RETURNS TABLE
(
  DEPARTMENT_ID    INT,
  DEPARTMENT_NAME  VARCHAR( 40 ),
  MANAGER_ID       INT,
  LOCATION_ID      INT
)
LANGUAGE JAVA
PARAMETER STYLE DERBY_JDBC_RESULT_SET
READS SQL DATA
EXTERNAL NAME 'vmware.au.se.sqlfire.derby.DepartmentTable.read'; 

6. Log into the distributed system using the CLI and run the SQL below to insert data into the "departments" table in SQLFire using the function we created at #5.
insert into departments
select s.*
FROM TABLE (externalDepartments()) s;

7. Verify we now have our departments table in SQLFire with data.
[Fri Mar 16 08:55:40 papicella@:~/sqlfire/vFabric_SQLFire_101/pasdemos/oraclehr ] $ sqlf
sqlf version 10.4
sqlf> connect client 'localhost:1527';
sqlf> select * from departments;
DEPARTMENT&|DEPARTMENT_NAME                         |MANAGER_ID |LOCATION_ID
----------------------------------------------------------------------------
230        |IT Helpdesk                             |NULL       |1700       
120        |Treasury                                |NULL       |1700       
10         |Administration                          |200        |1700       
240        |Government Sales                        |NULL       |1700       
130        |Corporate Tax                           |NULL       |1700       
20         |Marketing                               |201        |1800       
250        |Retail Sales                            |NULL       |1700       
140        |Control And Credit                      |NULL       |1700       
30         |Purchasing                              |114        |1700       
260        |Recruiting                              |NULL       |1700       
150        |Shareholder Services                    |NULL       |1700       
40         |Human Resources                         |203        |2400       
270        |Payroll                                 |NULL       |1700       
160        |Benefits                                |NULL       |1700       
50         |Shipping                                |121        |1500       
170        |Manufacturing                           |NULL       |1700       
60         |IT                                      |103        |1400       
180        |Construction                            |NULL       |1700       
70         |Public Relations                        |204        |2700       
190        |Contracting                             |NULL       |1700       
80         |Sales                                   |145        |2500       
200        |Operations                              |NULL       |1700       
90         |Executive                               |100        |1700       
210        |IT Support                              |NULL       |1700       
100        |Finance                                 |108        |1700       
220        |NOC                                     |NULL       |1700       
110        |Accounting                              |205        |1700       

27 rows selected 

More info on derby-style functions can be found here.

http://db.apache.org/derby/docs/10.4/devguide/cdevspecialtabfuncs.html

Friday, 17 February 2012

Using SQLFire sqlf CLI to connect to Oracle from a MAC OS-X Lion

Being on a MAC OS-X lion there is no client install for oracle to give me SQLPlus for example. BUT I can use sqlf command line client to connect to oracle to give me the ability to execute SQL much like SQLPLus. Handy indeed even if it isn't exactly SQLPlus it's good ienough for what I need when accessing my Oracle database from my MAC OS-X lion given thier is no oracle client for 11g on that platform.

It's done as shown below.

1. Add oracle JDBC driver to your classpath

export CLASSPATH=/Users/papicella/vmware/jdbcdrivers/11.2/ojdbc6.jar

2. Setup PATH to include SQLFire in your path as shown below.

export PATH=/Users/papicella/vmware/software/sqlfire/vFabric_SQLFire_101/bin:$PATH

3. Run some Oracle SQL as shown below using sql which shows how to load the driver, connect and run some SQL.

[Fri Feb 17 09:04:11 papicella@:~ ] $ sqlf
sqlf version 10.4
sqlf> driver 'oracle.jdbc.OracleDriver';
sqlf> connect 'jdbc:oracle:thin:scott/tiger@172.16.101.70:1521/linux11gr2';
sqlf> select to_char(sysdate, 'dd-MON-yyyy HH24:MI:SS') "Todays Date" from dual;
Todays Date                  
-----------------------------
17-FEB-2012 09:03:34         

1 row selected
sqlf> select * from dept;
DEPTNO                |DNAME         |LOC          
---------------------------------------------------
10                    |ACCOUNTING    |NEW YORK     
20                    |RESEARCH      |DALLAS       
30                    |SALES         |CHICAGO      
40                    |OPERATIONS    |BOSTON       

4 rows selected
sqlf> select * from v$version;
BANNER                                                                          
--------------------------------------------------------------------------------
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production    
PL/SQL Release 11.2.0.2.0 - Production                                          
CORE 11.2.0.2.0 Production                                                      
TNS for Linux: Version 11.2.0.2.0 - Production                                  
NLSRTL Version 11.2.0.2.0 - Production                                          

5 rows selected
sqlf>

Tuesday, 7 February 2012

How to offload data from an existing data store and loading it into SQLFire

A common requirement for using SQLFire is offloading data from an existing data store and loading it into SQLFire. This example show how you can load data into a SQLFire distributed system using Spring Batch from a CSV file and then how to present the data from a simple Spring MVC application. Finally the data is persisted to each SQLFire member to ensure we only need to load the data once and it will always be retained upon restarts of the distributed system.

The full demo can be viewed on the VMware vFabric SQLFire blog using the link below.

http://blogs.vmware.com/sqlfire/2012/02/sqlfire-demo-loading-the-afl-2012-fixture-data.html

Sunday, 5 February 2012

SQLFire locator equals Load Balancing + Failover

In this demo we show how not only does a locator provide load balancing among the SQLFire members from client connections but also high availability should a member crash or perhaps be brought down. This is illustrated below when using a JDBC Connection pool.

1. First we have started a locator and 2 SQLFire members as shown below.

Locator:
sqlf locator start -peer-discovery-address=localhost -peer-discovery-port=41111 -client-bind-address=localhost -client-port=1527

Members:

sqlf server start -server-groups=MYGROUP -locators=localhost[41111] -client-bind-address=localhost -client-port=1528 -dir=server1 &
sqlf server start -server-groups=MYGROUP -locators=localhost[41111] -client-bind-address=localhost -client-port=1529 -dir=server2 &

2. You can verify this by issuing a query as shown below , prior to running a JDBC Connection pool test.

Note: Notice how we connect to the locator client hostname/port rather then the individual SQLFire members.
[Sun Feb 05 21:38:50 papicella@:~/vmware/software/sqlfire/vFabric_SQLFire_101/pasdemos/simpledemo ] $ sqlf
sqlf version 10.4
sqlf> connect client 'localhost:1527';
sqlf> select substr(netservers, 1, 30) as "Netservers", kind from sys.members;
Netservers                    |KIND            
-----------------------------------------------
localhost/127.0.0.1[1529]     |datastore(norma&
localhost/127.0.0.1[1528]     |datastore(norma&
localhost/127.0.0.1[1527]     |locator(normal) 

3 rows selected

3. Here is our JDBC Connection Pool which shows that it's connecting to the locator itself to establish JDBC Conections.
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

  <bean id="sqlfireUcpDataSource" class="oracle.ucp.jdbc.PoolDataSourceFactory" factory-method="getPoolDataSource">
    <property name="URL" value="jdbc:sqlfire://localhost:1527/" />
    <property name="connectionFactoryClassName" value="com.vmware.sqlfire.jdbc.ClientDriver" />
    <property name="connectionPoolName" value="SQLFIRE_UCP_POOL" />
    <property name="minPoolSize" value="5" />
    <property name="maxPoolSize" value="20" />
    <property name="initialPoolSize" value="5" />
  </bean>
  
</beans>

4. At this point we create a test class which simply gets 5 connections and displays which SQLFire member we are connected to. So the process is as follows to test both load balancing and high availability when a member disappears, ensuring that existing connections fail over to the remaining SQLFire member.

- start JDBC Connection pool
- get 5 connections
- display each connection server side SQLFire member (It should load balance between our servers in this case we have 2 of them)
- Sleep for 10 seconds
- get 5 connections (verify that indeed we are only connected to the surviving member)

5. Output as follows.

Feb 5, 2012 9:59:26 PM org.springframework.context.support.AbstractApplicationContext prepareRefresh
INFO: Refreshing org.springframework.context.support.ClassPathXmlApplicationContext@657d5d2a: startup date [Sun Feb 05 21:59:26 EST 2012]; root of context hierarchy
Feb 5, 2012 9:59:26 PM org.springframework.beans.factory.xml.XmlBeanDefinitionReader loadBeanDefinitions
INFO: Loading XML bean definitions from class path resource [application-config-SQLFIRE.xml]
Feb 5, 2012 9:59:26 PM org.springframework.beans.factory.support.DefaultListableBeanFactory preInstantiateSingletons
INFO: Pre-instantiating singletons in org.springframework.beans.factory.support.DefaultListableBeanFactory@62da3a1e: defining beans [sqlfireUcpDataSource]; root of factory hierarchy
Feb 5, 2012 9:59:27 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: $Proxy0@1dfd868
Feb 5, 2012 9:59:27 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: Getting 5 connections from pool
Connection 0 [id=192-168-1-4.tpgi.com.au(1426):55091/50932, netserver=localhost/127.0.0.1[1528]]
Connection 1 [id=192-168-1-4.tpgi.com.au(1426):55091/50932, netserver=localhost/127.0.0.1[1528]]
Connection 2 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]
Connection 3 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]
Connection 4 [id=192-168-1-4.tpgi.com.au(1426):55091/50932, netserver=localhost/127.0.0.1[1528]]

Feb 5, 2012 9:59:27 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: Closing all connections from pool
Feb 5, 2012 9:59:27 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: ** Pool Details **
NumberOfAvailableConnections: 5
BorrowedConnectionsCount: 0


Sleeping for 20 seconds, shutdown a SQLFire server at this point...

6. Shutdown a server as shown below , before the existing program wakes up..

[Sun Feb 05 21:45:34 papicella@:~/vmware/software/sqlfire/vFabric_SQLFire_101/pasdemos/simpledemo ] $ sqlf server stop -dir=server1
The SQLFire Server has stopped.

7. Verify final output as follows

....

Feb 5, 2012 9:59:47 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: $Proxy0@580754fc
Feb 5, 2012 9:59:47 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: Getting 5 connections from pool
Connection 0 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]
Connection 1 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]
Connection 2 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]
Connection 3 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]
Connection 4 [id=192-168-1-4.tpgi.com.au(1425):52022/50935, netserver=localhost/127.0.0.1[1529]]

Feb 5, 2012 9:59:47 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: Closing all connections from pool
Feb 5, 2012 9:59:47 PM vmware.au.sqlfire.pool.TestSQLFireJDBCPool run
INFO: ** Pool Details **
NumberOfAvailableConnections: 5
BorrowedConnectionsCount: 0


As you can see from the output above we have switched over all existing connections in our JDBC pool to use the remaining SQLFire member only.

The test class used here is as follows.
package vmware.au.sqlfire.pool;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;

import oracle.ucp.jdbc.PoolDataSource;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

public class TestSQLFireJDBCPool 
{
 private Logger log = Logger.getLogger("com.vmware.sqlfire");
 private ApplicationContext context;
 private static final String BEAN_NAME = "sqlfireUcpDataSource";
 private PoolDataSource pds;
 
 public TestSQLFireJDBCPool() 
 {
  context = new ClassPathXmlApplicationContext("application-config-SQLFIRE.xml");
  pds = (PoolDataSource) context.getBean(BEAN_NAME);  
 }

 public void run()
 {
  Connection conn = null;
  List<Connection> connections = new ArrayList<Connection>();
  
  try 
  {
   // ensure pool is started by getting a connection object
   conn = pds.getConnection();
   log.info(conn.toString());
   conn.close();
   
   log.info("Getting 5 connections from pool");
   
   for (int i = 0; i < 5; i++)
   {
    connections.add(pds.getConnection());
   }

   for (int i = 0; i < 5; i++)
   {
    getInstanceDetails(connections.get(i), i);
   }
   
   log.info("Closing all connections from pool");
   
   for (Connection connection: connections)
   {
     connection.close(); 
   }
   
   log.info(displayPoolDetails());
   
 
  } 
  catch (SQLException e) 
  {
   // TODO Auto-generated catch block
   e.printStackTrace();
  } 
  finally
  {
   connections = null;
  }
 }

 public void getInstanceDetails (Connection conn, int i) throws SQLException
 {
  Statement stmt = null;
  PreparedStatement pstmt = null;
  ResultSet rset = null;
  String idSql = "select dsid() from sysibm.sysdummy1";
  String netSeverSql = "select netservers from sys.members where id = ?";
  
  stmt = conn.createStatement();
  rset = stmt.executeQuery(idSql);
  rset.next();
  
  String id = rset.getString(1);
  
  rset = null;
  pstmt = conn.prepareStatement(netSeverSql);
  pstmt.setString(1, id);
  rset = pstmt.executeQuery();
  rset.next();
  
  String netServer = rset.getString(1);
  
  System.out.println
    (String.format("Connection %s [id=%s, netserver=%s]", i, id, netServer));
  
  rset.close();
  stmt.close();
  pstmt.close();
  
 }
   
    public String displayPoolDetails () throws SQLException
    {
      StringBuffer sb = new StringBuffer();
      
      sb.append("** Pool Details **\n");
      sb.append("NumberOfAvailableConnections: " +
                         pds.getAvailableConnectionsCount());
      sb.append("\nBorrowedConnectionsCount: " +
                         pds.getBorrowedConnectionsCount());
      sb.append("\n");
      
      return sb.toString();
    }
    
 /**
  * @param args
  * @throws InterruptedException 
  */
 public static void main(String[] args) throws InterruptedException 
 {
  TestSQLFireJDBCPool test = new TestSQLFireJDBCPool();
  test.run();
        System.out.println("\nSleeping for 20 seconds, shutdown a SQLFire server at this point...\n");
  Thread.sleep(20000);
  test.run();
 }

}